From aed1bf1a352acd4f5ffd2a81e23d2bb6e8e9b636 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 19 Aug 2026 19:20:21 -1000 Subject: [PATCH 001/660] bpf: sched_ext: Mark ops argument container pointer fields as trusted Walking an unannotated pointer field of a trusted struct yields a bare PTR_TO_BTF_ID in non-sleepable programs, which kfuncs and helpers accept, but PTR_UNTRUSTED in sleepable programs, which they reject. This gets in the way of making ops.init_task() sleepable, which schedulers want for allocations. For example, passing args->cgroup into bpf_cgrp_storage_get() then fails verification and the only recourse is round-tripping through the cgroup ID with bpf_cgroup_from_id(). The pointer fields in the sched_ext ops argument containers are all pinned by the callers for the duration of the ops calls and are never NULL. Add them to the verifier's trusted-fields whitelist so that they are PTR_TRUSTED in both sleepable and non-sleepable programs. Fixes: 8195136669661 ("sched_ext: Add cgroup support") Signed-off-by: Tejun Heo Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/95d7ccc17681aa3a4a2eeb1b073f00f7@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e421ea2b80c3..821b47ac75c5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5877,6 +5877,28 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; @@ -5921,6 +5943,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } From ee66e3796b80b8d95430ad5d765b4ab24be06d52 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:11 -0400 Subject: [PATCH 002/660] selftests/bpf: libarena: Normalize SPDX headers across files Normalize SPDX in libarena for missing/inconsistent headers. There are currently three files missing headers and two of them (both originally authored by Kartikeya, CC'ed below) that are GPLv2 only. This patch ensures all files in libarena have a dual GPLv2/BSD-2-Clause license in the example of other non- kernel BPF codebases, e.g., libbpf. Add the customary BPF selftests/ SPDX tag to the bpf_may_goto.h file that holds the may_goto and can_loop macros for BPF code. The original authors of the code from when it was still in bpf_experimental.h are CC'ed below. Also add the missing SPDX license to the libarena bitmap.h header. For bpf_arena_spinlock.h and bpf_atomic.h, move the license to Dual GPLv2/BSD-2-Clause. This ensures all components have the same license, with the same rationale as libbpf in tools/lib. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-2-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/libarena/include/bpf_arena_spin_lock.h | 2 +- tools/testing/selftests/bpf/libarena/include/bpf_atomic.h | 2 +- tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h | 1 + tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h | 1 + .../testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c | 3 +++ 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index 71d9db610263..055b5892cb63 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ARENA_SPIN_LOCK_H #define BPF_ARENA_SPIN_LOCK_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index 43c306e17f19..65582b2010ad 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ATOMIC_H #define BPF_ATOMIC_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h index 9ba90689d6ba..b32a420d4e1a 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once /* diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index e2431ea6fdd6..8c5936ae9958 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once #define BITS_PER_BYTE 8 diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c index 76319a529f02..e66b3a26ca3d 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + #include #include From db00d61fa433d40fe444d8887d143b895372b274 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:12 -0400 Subject: [PATCH 003/660] selftests/bpf: libarena: Inline nonatomic bitmap operations The libarena code currently defines the non-atomic bitmap set/get operations as __weak functions, in accordance with the libarena coding style. This, however, is significant overhead to call functions that span single-digit instructions. Make an exception and expose the getters/setters as static inline functions in the header. Since the function body is now inlined into the caller, mark reads/writes with READ_ONCE()/WRITE_ONCE() to prevent compiler optimizations from breaking code that locklessly polls the bitmap. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-3-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/libarena/include/libarena/bitmap.h | 29 +++++++++++++++++-- .../selftests/bpf/libarena/src/bitmap.bpf.c | 18 ------------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index 8c5936ae9958..163e2b83d943 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -1,6 +1,8 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once +#include + #define BITS_PER_BYTE 8 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) @@ -16,11 +18,8 @@ struct arena_bitmap { struct arena_bitmap __arena *bmp_alloc(size_t bits); void bmp_free(struct arena_bitmap __arena *bmp); -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); @@ -33,3 +32,27 @@ void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); + +static __always_inline +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word |= BIT_MASK(bit); +} + +static __always_inline +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word &= ~BIT_MASK(bit); +} + +static __always_inline +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 word = READ_ONCE(bmp->bits[BIT_WORD(bit)]); + + return word & BIT_MASK(bit); +} diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c index 5ff8e688ddc7..0390f20ce366 100644 --- a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -34,24 +34,6 @@ void bmp_free(struct arena_bitmap __arena *bmp) arena_free(bmp); } -__weak -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); -} - -__weak -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); -} - -__weak -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); -} - __weak bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) { From 620e1e2b4377f03834894f86ac85ec865983ad0a Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:13 -0400 Subject: [PATCH 004/660] selftests/bpf: libarena: Disable IRQs during allocation The libarena buddy allocator currently uses arena_spin_lock/unlock to protect its internal data structures in its critical section. These locks disable preemption, but not IRQs. This in turns can cause ABBA deadlocks when an allocation/free operation gets an IRQ while in the critical section, and can only resume after another operation that in turn blocks on the buddy lock. We have concretely seen this with sched_ext schedulers: a) Task 1 on CPU A attempts an allocation during initialization, which is done without holding an rq lock. The task takes an IRQ in the middle of the allocation. b) Task 2 on CPU B exits. It attempts to take the buddy allocator lock during its sched-ext state teardown, and blocks on the spinlock. It does so while holding CPU B's rq lock. c) The scheduler run on CPU A and attempts to move tasks from CPU B's rq to CPU A's rq before resuming running Task 1. This requires B's rq lock, which requires Task 2 to take the buddy allocator lock first. Fix this by disabling IRQs when taking the buddy lock. We use the already existing arena_spin_[lock_irqsave, unlock_irqrestore] calls for this. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-4-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/libarena/src/buddy.bpf.c | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c index c674ee5cfcc1..2490ab1396de 100644 --- a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c @@ -5,6 +5,8 @@ #include #include +#include + /* * Buddy allocator arena-based implementation. * @@ -45,15 +47,8 @@ enum { BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE }; -static inline int buddy_lock(struct buddy __arena *buddy) -{ - return arena_spin_lock(&buddy->lock); -} - -static inline void buddy_unlock(struct buddy __arena *buddy) -{ - arena_spin_unlock(&buddy->lock); -} +#define buddy_lock(buddy, flags) (arena_spin_lock_irqsave(&(buddy)->lock, (flags))) +#define buddy_unlock(buddy, flags) (arena_spin_unlock_irqrestore(&(buddy)->lock, (flags))) /* * Reserve part of the arena address space for the allocator. We use @@ -385,6 +380,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) { u64 order, ord, min_order, max_order; struct buddy_chunk __arena *chunk; + unsigned long flags; size_t left; int power2; u64 vaddr; @@ -416,7 +412,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) return NULL; } - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { /* * We cannot reclaim the vaddr space, but that is ok - this * operation should always succeed. The error path is to catch @@ -520,7 +516,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) arena_stderr( "chunk has size of 0x%lx bytes (left %lx bytes)\n", sizeof(*chunk), left); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -531,7 +527,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) order = (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? power2 - BUDDY_MIN_ALLOC_SHIFT : 0; if (idx_set_allocated(chunk, idx, true)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -547,7 +543,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) */ min_order = left ? order + 1 : order; if (add_leftovers_to_freelist(chunk, idx, min_order, max_order)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -556,7 +552,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) max_order = order; } - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk; } @@ -564,6 +560,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) __weak int buddy_init(struct buddy __arena *buddy) { struct buddy_chunk __arena *chunk; + unsigned long flags; int ret; if (!asan_ready()) @@ -579,7 +576,7 @@ __weak int buddy_init(struct buddy __arena *buddy) chunk = buddy_chunk_get(buddy); - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); return -EINVAL; } @@ -591,7 +588,7 @@ __weak int buddy_init(struct buddy __arena *buddy) /* Put the chunk at the beginning of the list. */ buddy->first_chunk = chunk; - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk ? 0 : -ENOMEM; } @@ -730,9 +727,10 @@ static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int ord */ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order) { + unsigned long flags; u64 address; - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return (u64)NULL; @@ -745,7 +743,7 @@ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_ address = buddy_chunk_alloc(buddy->first_chunk, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return (u64)address; } @@ -754,6 +752,7 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) { void __arena *address = NULL; struct buddy_chunk __arena *chunk; + unsigned long flags; int order; if (!buddy) @@ -765,11 +764,11 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) return NULL; } - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return NULL; address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); if (address) goto done; @@ -880,6 +879,7 @@ static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) { + unsigned long flags; int ret; if (!buddy) @@ -889,13 +889,13 @@ __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) if (!addr) return 0; - ret = buddy_lock(buddy); + ret = buddy_lock(buddy, flags); if (ret) return ret; ret = buddy_free_unlocked(buddy, (u64)addr); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return ret; } From 283cb6513c881ccc6ff81f3da75d55e1ab1fe6fc Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:14 -0400 Subject: [PATCH 005/660] selftests/bpf: libarena: Add calloc() call The arena memory allocator currently does not provide the option to zero-initialize the allocated memory. Provide a calloc() operation that calls memset() on the returned memory. Reuse naive memset implementation already present in the arena ASAN code for now. Subsequent patches will optimize the function. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-5-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/libarena/include/libarena/common.h | 18 +++++++++++++ .../selftests/bpf/libarena/src/asan.bpf.c | 21 ++------------- .../selftests/bpf/libarena/src/common.bpf.c | 27 ++++++++++++++++++- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index 931ace9a49e2..d32a51ff5e7f 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -49,6 +49,7 @@ extern volatile u64 asan_violated; int arena_fls(__u64 word); void __arena *arena_malloc(size_t size); +void __arena *arena_calloc(size_t ncount, size_t size); void arena_free(void __arena *ptr); /* @@ -61,6 +62,23 @@ void arena_free(void __arena *ptr); */ #define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0) +/* + * BPF does not currently support the memset intrinsics. for large + * sequential copies, or assignments of large data structures, + * the frontend will generate an intrinsic that causes the BPF + * backend to exit due to a missing implementation. Provide + * implementations for the intrinsic. + */ +static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) +{ + size_t i; + + for (i = zero; i < size && can_loop; i++) + dst[i] = val; + + return 0; +} + #else /* ! __BPF__ */ #include diff --git a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c index 5135d5c72a46..de656b69d13a 100644 --- a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c @@ -103,23 +103,6 @@ volatile bool asan_inited = false; */ volatile bool asan_report_once = false; -/* - * BPF does not currently support the memset/memcpy/memcmp intrinsics. - * For large sequential copies, or assignments of large data structures, - * the frontend will generate an intrinsic that causes the BPF backend - * to exit due to a missing implementation. Provide a simple implementation - * just for memset to use it for poisoning/unpoisoning the map. - */ -__weak int asan_memset(s8 __arena *dst, s8 val, size_t size) -{ - size_t i; - - for (i = zero; i < size && can_loop; i++) - dst[i] = val; - - return 0; -} - /* Validate a 1-byte access, always within a single byte. */ static __always_inline bool memory_is_poisoned_1(s8 __arena *addr) { @@ -422,7 +405,7 @@ __hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, val, len); + arena_memset(shadow, val, len); return 0; } @@ -463,7 +446,7 @@ __hidden __noasan int asan_unpoison(void __arena *addr, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, 0, len); + arena_memset(shadow, 0, len); /* * If we are allocating a non-granule aligned region, we need to adjust diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 41b1de3452fe..d1af60b81838 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + #include #include #include @@ -55,10 +57,33 @@ __weak void __arena *arena_malloc(size_t size) return buddy_alloc(&buddy, size); } +__weak void __arena *arena_calloc(size_t ncount, size_t size) +{ + void __arena *mem; + size_t total; + + /* + * Ideally we'd be using __builtin_mul_overflow here, + * but the BPF compiler backend doesn't implement __multi3. + * There are ways to optimize the division away from the + * overflow check, but any costs are dwarfed by the + * buddy_alloc() call. Keep it simple for now. + */ + if (unlikely(ncount && size >= ULLONG_MAX / ncount)) + return NULL; + + total = ncount * size; + + mem = buddy_alloc(&buddy, total); + if (likely(mem)) + arena_memset(mem, 0, total); + + return mem; +} + __weak void arena_free(void __arena *ptr) { buddy_free(&buddy, ptr); } - char _license[] SEC("license") = "GPL"; From 047099824cbe646c590478550b511530a70969f2 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:15 -0400 Subject: [PATCH 006/660] selftests/bpf: libarena: Add a benchmark for malloc()/calloc() Add a benchmark for measuring the performance of the malloc()/calloc() arena allocator calls. This is useful as a basic allocator performance check that we can easily expand later. As with the regular arena allocator, focus on sub-page allocations that cannot be satisfied efficiently with the BPF arena page allocation code. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-6-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 18 +- tools/testing/selftests/bpf/bench.c | 6 + .../selftests/bpf/benchs/bench_libarena.c | 210 ++++++++++++++++++ .../bpf/benchs/run_bench_libarena.sh | 31 +++ tools/testing/selftests/bpf/libarena/Makefile | 30 ++- .../bpf/libarena/benchs/bench_malloc.bpf.c | 51 +++++ 6 files changed, 334 insertions(+), 12 deletions(-) create mode 100644 tools/testing/selftests/bpf/benchs/bench_libarena.c create mode 100755 tools/testing/selftests/bpf/benchs/run_bench_libarena.sh create mode 100644 tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 2b2f93dec474..5f1a3bfc0569 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -826,18 +826,22 @@ LIBARENA_MAKE_ARGS = \ BPF_TARGET_ENDIAN="$(BPF_TARGET_ENDIAN)" \ Q="$(Q)" -LIBARENA_BPF_DEPS := $(wildcard libarena/Makefile \ - libarena/include/* \ - libarena/include/libarena/* \ - libarena/src/* \ - libarena/selftests/* \ - libarena/*.bpf.o) +LIBARENA_COMMON_DEPS := $(wildcard libarena/Makefile \ + libarena/include/* \ + libarena/include/libarena/* \ + libarena/src/*) +LIBARENA_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/selftests/*) +LIBARENA_BENCH_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/benchs/*) LIBARENA_SKEL := libarena/libarena.skel.h +LIBARENA_BENCH_SKEL := libarena/libarena_bench.skel.h $(LIBARENA_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena.skel.h $(LIBARENA_MAKE_ARGS) +$(LIBARENA_BENCH_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BENCH_BPF_DEPS) | $(LIBARENA_SKEL) + +$(MAKE) -C libarena benchmarks $(LIBARENA_MAKE_ARGS) + ifneq ($(CLANG_HAS_ARENA_ASAN),) LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h CFLAGS += -DHAS_BPF_ARENA_ASAN @@ -987,6 +991,7 @@ $(OUTPUT)/bench_sockmap.o: $(OUTPUT)/bench_sockmap_prog.skel.h $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_trie_map.skel.h $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_libarena.o: $(LIBARENA_BENCH_SKEL) $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm @@ -1014,6 +1019,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_bpf_timing.o \ $(OUTPUT)/bench_bpf_nop.o \ $(OUTPUT)/bench_xdp_lb.o \ + $(OUTPUT)/bench_libarena.o \ $(OUTPUT)/usdt_1.o \ $(OUTPUT)/usdt_2.o \ # diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 465233ea5f79..d5f494184f9d 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -288,6 +288,7 @@ extern struct argp bench_crypto_argp; extern struct argp bench_sockmap_argp; extern struct argp bench_lpm_trie_map_argp; extern struct argp bench_xdp_lb_argp; +extern struct argp bench_libarena_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -306,6 +307,7 @@ static const struct argp_child bench_parsers[] = { { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 }, { &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 }, { &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 }, + { &bench_libarena_argp, 0, "libarena allocator benchmark", 0 }, {}, }; @@ -585,6 +587,8 @@ extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; extern const struct bench bench_bpf_nop; extern const struct bench bench_xdp_lb; +extern const struct bench bench_libarena_malloc; +extern const struct bench bench_libarena_calloc; static const struct bench *benchs[] = { &bench_count_global, @@ -669,6 +673,8 @@ static const struct bench *benchs[] = { &bench_lpm_trie_free, &bench_bpf_nop, &bench_xdp_lb, + &bench_libarena_malloc, + &bench_libarena_calloc, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_libarena.c b/tools/testing/selftests/bpf/benchs/bench_libarena.c new file mode 100644 index 000000000000..24e432244bf5 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_libarena.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include + +#include "bench.h" + +#include +#include +#include +#include + +#include "libarena/libarena_bench.skel.h" + +static struct { + __u64 alloc_size; + __u64 nallocs; +} args = { + .alloc_size = 64, + .nallocs = 10000, +}; + +static struct { + struct libarena_bench *skel; + int bench_fd; + int reset_fd; +} ctx; + +enum { + ARG_LIBARENA_ALLOC_SIZE = 12000, + ARG_LIBARENA_NALLOCS, +}; + +static const struct argp_option opts[] = { + { "alloc_size", ARG_LIBARENA_ALLOC_SIZE, "BYTES", 0, + "Size of each arena allocation" }, + { "nallocs", ARG_LIBARENA_NALLOCS, "ITERS", 0, + "Number of allocation per measurement" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + unsigned long value; + + switch (key) { + case ARG_LIBARENA_ALLOC_SIZE: + value = strtoull(arg, NULL, 10); + if (!value || value >= UINT_MAX) { + fprintf(stderr, "invalid alloc_size: %ld", value); + argp_usage(state); + } + args.alloc_size = value; + break; + case ARG_LIBARENA_NALLOCS: + args.nallocs = strtoull(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_libarena_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } + + if (env.producer_cnt != 1) { + fprintf(stderr, "benchmark supports exactly one producer\n"); + exit(1); + } +} + +static void setup_common(void) +{ + struct arena_alloc_reserve_args reserve_args = { + .nr_pages = ARENA_RESERVE_PAGES_DFL, + }; + int err; + + setup_libbpf(); + + ctx.skel = libarena_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open and load skeleton\n"); + exit(1); + } + + err = libarena_run_prog_args( + bpf_program__fd(ctx.skel->progs.arena_alloc_reserve), + &reserve_args, sizeof(reserve_args)); + if (err) { + fprintf(stderr, "failed to reserve arena pages: %d\n", err); + exit(1); + } + + err = libarena_run_prog( + bpf_program__fd(ctx.skel->progs.arena_buddy_reset)); + if (err) { + fprintf(stderr, "failed to initialize arena allocator: %d\n", err); + exit(1); + } + + ctx.skel->bss->bench_alloc_size = args.alloc_size; + ctx.skel->bss->bench_nallocs = args.nallocs; + ctx.reset_fd = bpf_program__fd(ctx.skel->progs.arena_buddy_reset); +} + +static void malloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_malloc); +} + +static void calloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_calloc); +} + +static void *producer(void *input) +{ + int err; + + while (true) { + err = libarena_run_prog(ctx.bench_fd); + if (err) { + fprintf(stderr, "libarena benchmark failed: %d\n", err); + exit(1); + } + + err = libarena_run_prog(ctx.reset_fd); + if (err) { + fprintf(stderr, "libarena alloc reset failed: %d\n", err); + exit(1); + } + } + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->duration_ns = atomic_swap(&ctx.skel->bss->bench_duration_ns, 0); + res->hits = atomic_swap(&ctx.skel->bss->bench_hits, 0); +} + +static void report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double latency_ns = 0.0; + + if (res->hits) + latency_ns = res->duration_ns / (double)res->hits; + + printf("Iter %3d (%7.3lfus): latency %8.3lf ns/op (%ld allocations)\n", + iter, (delta_ns - 1000000000) / 1000.0, latency_ns, res->hits); +} + +static void report_final(struct bench_res res[], int res_cnt) +{ + unsigned long duration_ns = 0; + long hits = 0; + int i; + + for (i = 0; i < res_cnt; i++) { + duration_ns += res[i].duration_ns; + hits += res[i].hits; + } + + if (!hits || !res_cnt) { + printf("Summary: no runs measured\n"); + return; + } + + printf("Summary: %.3lf ns/op, %.0lf invocations for %u allocations/invocation)\n", + duration_ns / (double)hits, hits / (double)res_cnt, + ctx.skel->bss->bench_nallocs); +} + +const struct bench bench_libarena_malloc = { + .name = "libarena-malloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = malloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; + +const struct bench bench_libarena_calloc = { + .name = "libarena-calloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = calloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh new file mode 100755 index 000000000000..10afe4d52ebf --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +RUN_BENCH="./bench -d3 -q" + +summarize_libarena() +{ + local bench="$1" + local summary + + summary=$(printf '%s\n' "$2" | tail -n1) + summary=${summary#Summary: } + printf "%-20s %s\n" "$bench" "$summary" +} + +header "libarena sequential malloc\n" + +for size in 16 64 256 1024 4096; do +subtitle "allocation size: $size" +printf "\t-------------------\n" + for nallocs in 10 50 100 500 1000 5000 10000; do + summarize_libarena "malloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-malloc)" + summarize_libarena "calloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-calloc)" + done +done diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 5e2ab514805e..6d62eda34920 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -27,10 +27,17 @@ BPFDIR=$(abspath $(LIBARENA)/..) INCLUDE_DIR ?= $(BPFDIR)/tools/include LIBBPF_INCLUDE ?= $(INCLUDE_DIR) -# Scan src/ and selftests/ to generate the final binaries -LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c) -LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o)) -LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o)) +# Build selftests and benchmarks into separate BPF objects and skeletons. +LIBARENA_CORE_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) +LIBARENA_TEST_SOURCES = $(wildcard $(LIBARENA)/selftests/*.bpf.c) +LIBARENA_BENCH_SOURCES = $(wildcard $(LIBARENA)/benchs/*.bpf.c) + +LIBARENA_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=.bpf.o)) +LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=_asan.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=_asan.bpf.o)) +LIBARENA_BENCH_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_BENCH_SOURCES:.bpf.c=.bpf.o)) INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR) ifneq ($(INCLUDE_DIR),) @@ -58,12 +65,19 @@ override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie CFLAGS += $(INCLUDES) -vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests -vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests +vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs +vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs skeletons: libarena.skel.h libarena_asan.skel.h .PHONY: skeletons +benchmarks: libarena_bench.skel.h +.PHONY: benchmarks + +libarena_bench.skel.h: libarena_bench.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena_bench" > $@ + libarena_asan.skel.h: libarena_asan.bpf.o $(call msg,GEN-SKEL,libarena,$@) $(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@ @@ -80,6 +94,10 @@ libarena.bpf.o: $(LIBARENA_OBJECTS) $(call msg,GEN-OBJ,libarena,$@) $(Q)$(BPFTOOL) gen object $@ $^ +libarena_bench.bpf.o: $(LIBARENA_BENCH_OBJECTS) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + %_asan.bpf.o: %.bpf.c $(call msg,CLNG-BPF,libarena,$@) $(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@ diff --git a/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c new file mode 100644 index 000000000000..e06397dc9a40 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + +#include +#include + +u32 bench_alloc_size; +u32 bench_nallocs; +long bench_hits; +long bench_duration_ns; + +SEC("syscall") +int bench_malloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_malloc(bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} + +SEC("syscall") +int bench_calloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_calloc(1, bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} From 8be79805b06e23ebdb7321fe3cdb788e949c984e Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:16 -0400 Subject: [PATCH 007/660] selftests/bpf: libarena: Optimize and make public arena_memset Clang currently provides no __builtin_{memset, memcpy, memcmp} for its BPF backend. This is especially an issue for arena code that is more likely to do these operations on buffers with user-provided bounds. One example is the arena ASAN implementation that uses memset to update the shadow bitmap. Arena ASAN actually already has a naive implementation of this operation. Another user would be a calloc() call that has to zero the memory it returns. Introduce a more optimized version of the memset() operation for arena memory and make it public to all libarena users. The operation uses word-sized assignments to speed up the function for larger sizes. We expose the function through common.h to allow for inlining from the callers. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-7-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/libarena/Makefile | 1 + .../bpf/libarena/include/libarena/common.h | 57 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 6d62eda34920..91164d43bd61 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -60,6 +60,7 @@ override BPF_CFLAGS += -O2 -g override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers # Required for suppressing harmless vmlinux.h-related warnings. override BPF_CFLAGS += -Wno-missing-declarations +override BPF_CFLAGS += -fno-strict-aliasing override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index d32a51ff5e7f..a6220355a697 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -71,10 +71,63 @@ void arena_free(void __arena *ptr); */ static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) { + size_t headalign; + size_t tailalign; + u8 uval = (u8)val; + size_t val64; size_t i; - for (i = zero; i < size && can_loop; i++) - dst[i] = val; + /* + * Calculate how many bytes to the next word-aligned one. + * We get this by truncating the 2s complement of the + * pointer to the last 3 bits. Intuitively, since + * + * The N LSBs of dst and -dst add to 1 << N, which + * is why dst + (-dst) = 0x0ULL through overflow. So the + * last N = 3 bits of the negative are the number of + * bytes to align dst on the last 3 bits. + * + */ + headalign = -(u64)dst & (sizeof(u64) - 1); + if (!headalign || size < headalign) + goto ptraligned; + + for (i = zero; i < headalign && can_loop; i++) + dst[i] = uval; + + dst += headalign; + size -= headalign; + +ptraligned: + + /* + * Make a word with all bytes equal to the byte we are setting. + * Since 1 byte -> 2 hex digits. + * + * Shifting the value by a 0 bytes is equal to multiplication by 0x01 + * Shifting by 1 bytes is equal to multiplication by 0x01 << 8, + * ... + * Shifting by 7 bytes is equal to multiplication by 0x01 << 56. + * + * End operation to replicate the byte into all the bytes of a word + * is (since a | b = a + b when a & b == 0): + * + * val + val * (1UL << 8) + val * (1UL << 16) + .. + val * (1UL << 56) + * = val * (1UL << 56 + 1UL << 48 + ... + 1UL << 0) + * = val * (0x01UL << 56 | 0x01UL << 48 + ... + 1UL << 0) + * = val * 0x0101 0101 0101 0101 + */ + val64 = (u8)val * 0x0101010101010101ULL; + + /* Pointer is now aligned, use word-aligned assignments. */ + for (i = zero; i < size / sizeof(u64) && can_loop; i++) + ((u64 __arena *)dst)[i] = val64; + + /* Go back to byte-aligned for the tail. */ + tailalign = size % sizeof(u64); + dst += size - tailalign; + for (i = zero; i < tailalign && can_loop; i++) + dst[i] = uval; return 0; } From 4e3fb5bba680c1778e05a120c3793d22204f8fd4 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:18 +0000 Subject: [PATCH 008/660] bpf: Simplify cnum contains() by removing redundant branches Bearing in mind that a non-empty cnum represents an inclusive circular range in the corresponding unsigned integer domain, membership could be tested by checking whether the distance from the range base to the queried value is within the range size. This makes the explicit wrapping and non-wrapping cases in contains() unnecessary, since: v - cnum.base <= cnum.size is equivalent for both ordinary ranges and ranges that cross the unsigned wrap boundary. We've also run some benchmarks to compare the performance of the new implementation, for example, for empty ranges, overflowing ranges non-overflowing ranges. We've run the tests with O2 optimization. Results: | Test | contains | contains_new | | -------------------- | ----------- | ------------ | | empty | 1.511 ns/op | 1.479 ns/op | | non-overflow inside | 1.156 ns/op | 1.270 ns/op | | non-overflow outside | 1.053 ns/op | 1.258 ns/op | | overflow high side | 1.469 ns/op | 1.257 ns/op | | overflow low side | 1.469 ns/op | 1.270 ns/op | | overflow outside | 1.478 ns/op | 1.256 ns/op | | singleton inside | 1.049 ns/op | 1.256 ns/op | | singleton outside | 1.051 ns/op | 1.254 ns/op | | full range | 1.053 ns/op | 1.256 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves the soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-2-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317e3578..ac1f62a0195d 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -210,12 +210,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum) From b601a4ecbe672371776785bd7ac142af0ac395ac Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:24 +0000 Subject: [PATCH 009/660] bpf: Simplify cnum normalize() by dropping unnecessary comparisons In normalize(), there's no need to compare cnum.base with ST_MAX, because any cnum with size == UT_MAX represents the full unsigned domain independently of base, so normalize() can canonicalize all such values to base 0. Also, it's a no-op assigning 0 to cnum.base if the cnum is already unbounded, so we can remove the base != 0 check too. We've also run some benchmarks to compare the performance of the new implementation. We've run the tests with O2 optimization. Results: | Test | normalize | normalize_new | | ----------------------- | ----------- | ------------- | | normal small range | 1.255 ns/op | 1.217 ns/op | | full range normalized | 1.019 ns/op | 1.217 ns/op | | full range nonzero base | 1.031 ns/op | 1.221 ns/op | | empty | 1.023 ns/op | 1.217 ns/op | | signed max full range | 1.020 ns/op | 1.218 ns/op | | max base singleton | 1.218 ns/op | 1.219 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-3-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index ac1f62a0195d..09535edbbafc 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -181,7 +181,7 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } From 705249119d942e7228d9207800ddfa13e4601b19 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:32 +0000 Subject: [PATCH 010/660] bpf: Avoid redundant min()/max() in cnum signed bounds Simplify the smin() and smax() calculation by removing the integer min and max calculation on the non-overflowing branch. This is a valid simplification because when the arc does not cross the signed ST_MAX/ST_MIN boundary, the signed order of the endpoints is preserved, so base is the signed minimum and base + size is the signed maximum. Therefore, the min()/max() calls are redundant. We've also run benchmarks to compare the performance of this change. With O2 optimization we have: smin() results: | Test | smin | smin_new | | ------------------------ | ----------- | ----------- | | empty | 1.866 ns/op | 1.477 ns/op | | positive range | 1.870 ns/op | 1.248 ns/op | | negative range | 1.657 ns/op | 1.246 ns/op | | cross zero | 1.870 ns/op | 1.660 ns/op | | signed overflow boundary | 1.463 ns/op | 1.049 ns/op | | signed overflow range | 1.519 ns/op | 1.050 ns/op | | unsigned overflow range | 1.883 ns/op | 1.669 ns/op | | full range | 1.555 ns/op | 1.049 ns/op | smax() results: | Test | smax | smax_new | | ------------------------ | ----------- | ----------- | | empty | 1.462 ns/op | 1.456 ns/op | | positive range | 1.250 ns/op | 1.249 ns/op | | negative range | 1.259 ns/op | 1.297 ns/op | | cross zero | 1.471 ns/op | 1.475 ns/op | | signed overflow boundary | 1.056 ns/op | 1.260 ns/op | | signed overflow range | 1.049 ns/op | 1.259 ns/op | | unsigned overflow range | 1.470 ns/op | 1.477 ns/op | | full range | 1.072 ns/op | 1.263 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of these operations are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-4-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 09535edbbafc..30685e43de04 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* From da13c047cda6ef1bbc9355adb040d5b4eecea710 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:44 -0700 Subject: [PATCH 011/660] bpf: Factor check_global_ret_scalar_reg() out of the global return check check_global_subprog_return_code() verifies that a global subprogram returns void, an arena pointer, or register R0 holding a scalar value. Later patches in this series add 16-byte aggregate return support, whose second half is returned in R2 and needs the same validation. Factor the per-register check into check_global_ret_scalar_reg(env, regno) so that it can be reused for R2. No functional change. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055244.3294700-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 821b47ac75c5..5265563c8abf 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17479,37 +17479,45 @@ enforce_retval: return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + + if (subprog_returns_void(env, subprog)) + return 0; + + return check_global_ret_scalar_reg(env, BPF_REG_0); +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) From cf5b96d9554815780f864d6f0014ac384e9be59c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:49 -0700 Subject: [PATCH 012/660] bpf: Add helpers to describe the R0: R2 return register pair LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes: such a value comes back in the R0:R2 register pair, with R2 holding the upper half. See LLVM patches [1] and [2]. Later patches teach the JIT, live register analysis and the verifier itself about that convention. They need to answer the same question: does this subprogram return its value in a register pair? Add bpf_ret_reg_pair() up front so it can be used in subsequent patches. It is answered from a per-subprogram flag that bpf_compute_subprog_ret_regs() derives once from the BTF prototype. jit_requested only says that the JIT is enabled, not that it succeeded: bpf_fixup_call_args() falls back to the interpreter when bpf_jit_subprogs() fails with anything other than -EFAULT. So jit_required is still set once the pair is modelled, which turns that fallback into a load failure rather than a silent divergence from what was verified. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055249.3295283-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 7 ++++ kernel/bpf/verifier.c | 66 ++++++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5fad59fdab0d..953d8191563a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -819,6 +819,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; @@ -1055,6 +1057,11 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { u8 num_params; bool is_void; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5265563c8abf..2be5867d4412 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -385,27 +385,66 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} - return btf_type_is_void(type); +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; + + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -21185,6 +21224,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; From 34c3503cb29ad0f73d413b83a0586e53d832bb90 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:54 -0700 Subject: [PATCH 013/660] bpf: Wire up JIT support for 16-byte kfunc returns LLVM 23 returns an __int128, or a struct/union larger than 8 bytes and no larger than 16 bytes, in the BPF R0:R2 register pair. The previous patch added the shared helpers describing that convention; wire up the JIT side so that the second half of the return value actually lands in R2. Placing the second return half into R2 is possible on any JIT, but it needs architecture-specific JIT work. Rather than requiring every JIT to implement it at once, add a bpf_jit_supports_kfunc_ret_reg_pair() capability, defaulting to false in the generic core; an architecture opts in once its JIT handles the R0:R2 pair, and the remaining ones are left for future work. Only the x86-64, arm64 and riscv64 JITs opt in so far. On arm64 and riscv64 the native second return register is already BPF R2 (x1 in bpf2a64[] and a1 in regmap[] respectively), so the upper half needs no move at all, unlike x86-64's RDX->RSI. The lower half is covered by the move into BPF R0 that those JITs already emit after every call, from x0 into x8 and from a0 into a5. This has been tested on x86-64 and arm64. The riscv64 path is expected to work by the same register-mapping reasoning as arm64 but has not been tested. bpf_add_kfunc_call() also rejects a kfunc that is marked KF_FASTCALL and returns more than 8 bytes. The bpf_fastcall contract implemented by mark_fastcall_pattern_for_call() assumes a call clobbers R0 plus the registers holding its arguments, so a return in the R0:R2 pair would clobber an R2 the caller expects the fastcall pattern to preserve. Such a kfunc is rejected with -EOPNOTSUPP as well. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055254.3295610-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ arch/riscv/net/bpf_jit_comp64.c | 5 +++++ arch/x86/net/bpf_jit_comp.c | 27 ++++++++++++++++++++------- include/linux/filter.h | 1 + kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 12 ++++++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..3aa3ea0bc30b 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2388,6 +2388,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 74efe4b138d2..47c7bf431ba8 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -2121,6 +2121,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1a9fb530adc3..48429fae0641 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1689,17 +1689,12 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; int i; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { u8 flags = fm->arg_flags[i]; u32 reg = BPF_REG_1 + i; @@ -2644,6 +2639,8 @@ populate_extable: /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { LOAD_TAIL_CALL_CNT_PTR(stack_depth); @@ -2652,7 +2649,10 @@ populate_extable: if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; @@ -2666,6 +2666,14 @@ populate_extable: return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } @@ -4156,6 +4164,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a9bc6a848f2..6e746b0a0930 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..5db77d7915df 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2be5867d4412..48e31136a7df 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2916,6 +2916,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; From 5bbb242c0859024d6d0ba990c0476b33c7c0be71 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:59 -0700 Subject: [PATCH 014/660] bpf: Handle R2 as a return register in precision backtracking Precision backtracking treats only R0 as a return register at a call/return boundary, so once the verifier starts modeling R2 that way, marking the second half of such a return precise would trip the "unexpected regs" checks in backtrack_insn() and reject a valid program. Marking the upper half precise, for example by branching on it after a call to a static subprogram, walks backtracking into the callee and reaches its BPF_EXIT with R2 still set in the mask. Handle R2 like R0 in boundaries where a call defines the return registers. R2 differs from R0 in that it is an argument register as well, so it is part of the BPF_REGMASK_ARGS check and has to be cleared before that check rather than next to R0. Clear it unconditionally, rather than only where the callee or the kfunc really does return a pair. That gives up the "unexpected regs" assertion for R2, and in exchange keeps backtracking free of any BTF lookup. Nothing is lost: a callee that does not return a pair leaves the caller's R2 uninitialized, so the main verification pass has already rejected any program that reads it, and backtracking is never asked for its precision. At BPF_EXIT the return registers are sampled before the callback path clears R1-R5. That clear does not touch R0, but it does cover R2, and running it first would drop a pair return whenever the instruction following the call happens to be one that invokes a callback. Suggested-by: Eduard Zingerman Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055259.3295829-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/backtrack.c | 46 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694..653db80bcc47 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -423,6 +423,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +436,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -506,6 +508,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,7 +524,25 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise, r2_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + /* Sample the return registers before the callback + * handling below clears R1-R5. + */ + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. @@ -533,30 +555,22 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); + + bt_clear_reg(bt, BPF_REG_0); + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ From 991fb7823106bb2faa8a4871bc1d7c745201bac9 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:04 -0700 Subject: [PATCH 015/660] bpf: Account R2 of register-pair returns in live register analysis A BPF_EXIT of a subprogram returning a value larger than 8 bytes (a struct/union or an __int128) reads R2 as well as R0, since the second half of the return value is passed back in R2. compute_insn_live_regs() only marked R0 used at exit, so a callee's R2 could be considered dead and cleaned from checkpointed states, which would allow unsound state pruning. Mark R2 as read at the BPF_EXIT of a subprogram that does return a register pair. bpf_compute_live_registers() now loops over the subprograms and, for each, over the [start, end) instruction range from env->subprog_info[], so the return convention is queried once per subprogram through bpf_ret_reg_pair() rather than once per instruction. Marking R2 at every exit instead would be simpler, but R2 would then stay live backwards across any call that is not followed by a write to R2, which is nearly every program, and would needlessly hurt state pruning. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055304.3296353-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/liveness.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 74fc4b3f80d6..71f998c6eb88 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2060,7 +2060,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2073,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); u32 def = 0; u32 use = U32_MAX; @@ -2191,7 +2193,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; @@ -2228,8 +2230,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct insn_live_regs *state; int insn_cnt = env->prog->len; u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, j, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2258,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); From 0630ad00d96d9cf600c5c340cdecb363caa9d1cc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:09 -0700 Subject: [PATCH 016/660] bpf: Add verifier support for 16-byte returns in R0: R2 LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes. See LLVM patches [1] and [2]. Before LLVM 23 the BPF backend could not return these values at all. A by-value struct or union return (of any size) was rejected at compile time with: error: aggregate returns are not supported and an __int128 return failed later in the backend with: fatal error: error in backend: unable to allocate function return #1 Both are resolved in LLVM 23, which lowers such returns into the R0:R2 register pair. Model that pair at calls to global and static BPF subprograms and at kfunc calls: R2 is marked alongside R0 at the call, propagated out of a callee at its exit, and held to the same scalar-only and no-stack-pointer rules that already apply to R0. A struct returned by a kfunc must be composed of scalars, since its bytes reach the program as raw register contents and a pointer field would otherwise be laundered into a scalar. An extension program is the one caller of the convention that cannot take part in it: only R0 is checked at its exit, since that is the program exit code, so it has no way to hand back an upper half. Replacing a function whose return value is larger than 8 bytes is therefore rejected in btf_check_func_type_match(). That function compares btf_type->info, which carries no size for an int, so an 8 byte long and a 16 byte __int128 compared equal; sizes up to 8 bytes stay interchangeable, as they always have been, so this only rejects what the R0:R2 convention newly makes incompatible. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055309.3297684-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 6 ++++ kernel/bpf/verifier.c | 68 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index da36d4b9d31a..a10a070016be 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7677,6 +7677,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 48e31136a7df..930925fada30 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -415,6 +415,9 @@ static u32 ret_regs_cnt(u32 size) return size > 8 && size <= 16 ? 2 : 1; } +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) { const struct btf *btf = env->prog->aux->btf; @@ -9930,6 +9933,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn; + u32 i, nregs; target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); @@ -9985,9 +9989,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, caller->regs, ret_regs[i]); bpf_diag_mod_end(env); } @@ -10347,11 +10356,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10386,9 +10399,13 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ + /* + * return to the caller whatever the callee had in the + * return register(s) + */ bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; + for (i = 0; i < nregs; i++) + caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; bpf_diag_mod_end(env); } @@ -11323,6 +11340,19 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars. Both halves of a register pair are treated the same way. + */ +static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[i]); +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -13991,10 +14021,25 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so it must contain only scalars: + * otherwise a pointer field would be laundered into a scalar + * and escape provenance and reference tracking. + */ + if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + return -EINVAL; + } + mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -17562,11 +17607,20 @@ static int check_global_subprog_return_code(struct bpf_verifier_env *env) { struct bpf_func_state *cur_frame = cur_func(env); u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; if (subprog_returns_void(env, subprog)) return 0; - return check_global_ret_scalar_reg(env, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; } /* Bitmask with 1s for all caller saved registers */ From a2e98481a6396f919ad91ba9c2ff3ec61c4028b3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:14 -0700 Subject: [PATCH 017/660] bpf: Enable aggregate return types up to 16 bytes Relax btf_distill_func_proto() to accept a by-value struct or union that the R0:R2 convention added in earlier patches can carry: - a struct or union larger than 8 and up to 16 bytes, returned in the R0:R2 register pair, matching what LLVM emits for the BPF target; - a struct or union up to 8 bytes, returned in R0 alone. A >8 byte scalar (__int128) was already accepted and is unchanged. Everything else stays rejected: a return type larger than 16 bytes, and any type that __get_type_size() cannot return in registers at all (e.g. an array), which it already reports as ret < 0. btf_validate_return_type() is relaxed as well, so that it accepts a by-value struct or union up to 16 bytes in addition to void and scalars. With btf_distill_func_proto() and btf_validate_return_type() relaxed, the verifier, JIT, precision-backtracking and live-register support from the earlier patches becomes reachable: <=16 byte aggregate return values now work end to end. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055314.3298664-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/btf.c | 23 +++++++++++++++---- kernel/bpf/verifier.c | 18 +++++++-------- .../selftests/bpf/progs/exceptions_fail.c | 2 +- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 953d8191563a..004b06785521 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,6 +1488,8 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a10a070016be..91b8ce77f699 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7584,7 +7584,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7963,7 +7963,7 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { u32 tags = 0; int err; @@ -7986,6 +7986,19 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so it may only return scalars by value. A local + * function is verified inline, so a pointer field stays tracked + * and needs no such restriction. + */ + bool local_func = subprog && !is_global; + + if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + return 0; + } + return -EOPNOTSUPP; } @@ -8073,12 +8086,12 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", + "Global function %s() has unsupported return type. " + "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", tname); } return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 930925fada30..e036ae20bf6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11620,9 +11620,9 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11640,7 +11640,7 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } @@ -12039,7 +12039,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12055,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13111,7 @@ check_ok: break; } - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13653,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14032,7 +14032,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e5066..9708efb93683 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); From 7b7b511e0cf53d2602ba6119c9a52174d548ff26 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:20 -0700 Subject: [PATCH 018/660] selftests/bpf: Add C tests for 16-byte returns in R0: R2 Add selftests that exercise a 16-byte return value passed in the R0:R2 register pair, written in C so that they depend on the compiler lowering the register-pair return. The R0:R2 convention is only emitted by LLVM 23 and newer, and a by-value aggregate return does not compile at all before that, so the programs sit behind a __clang_major__ guard. An older compiler builds the dummy test in the #else branch instead, which keeps the object non-empty and says in its description why nothing was exercised. The kfunc tests are tagged __arch_x86_64/__arch_arm64 and skip elsewhere. Those are the architectures whose JIT advertises bpf_jit_supports_kfunc_ret_reg_pair(), which bpf_add_kfunc_call() requires before it accepts a kfunc returning more than 8 bytes, and they are also the only ones building the kfuncs. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055320.3300114-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_aggregate_ret.c | 178 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 18 ++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 9 + 4 files changed, 207 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 64ac49ad67e6..f7f94ccebce2 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" @@ -170,6 +171,7 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c new file mode 100644 index 000000000000..7851bade2b40 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +typedef unsigned __int128 u128; + +struct pair { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ + } parts; +}; + +static __noinline u128 make_i128(__u64 a, __u64 b) +{ + return ((u128)(a + b) << 64) | (a - b); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = make_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +static __noinline struct pair make_pair(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +__noinline struct pair make_pair_global(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair_global(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +static __noinline union upair make_upair(__u64 a, __u64 b) +{ + union upair p; + + p.halves[0] = a + b; + p.halves[1] = a - b; + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair p; + + p = make_upair(a, b); + if (p.parts.lo != a + b) + return 1; + if (p.parts.hi != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = bpf_kfunc_call_test_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_ret_pair p; + + p = bpf_kfunc_call_test_ret_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +#else + +SEC("socket") +__description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 9366a3c578f1..c4bc8e11c6e1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -954,6 +954,20 @@ __bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) return 0; } +#if defined(__x86_64__) || defined(__aarch64__) +__bpf_kfunc __int128 bpf_kfunc_call_test_i128(u64 a, u64 b) +{ + return (__int128)(((unsigned __int128)(a + b) << 64) | (a - b)); +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} +#endif /* __x86_64__ || __aarch64__ */ + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1487,6 +1501,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +#if defined(__x86_64__) || defined(__aarch64__) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +#endif BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 7d81070eefe7..426000f9a14f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -55,6 +55,11 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ + __u64 lo; + __u64 hi; +}; + struct prog_test_fail1 { void *p; int x; @@ -131,6 +136,10 @@ int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +#ifdef __SIZEOF_INT128__ +__int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; +#endif +struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; From 0b9fc795f83f5831de858d659de1097800ba7f18 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:25 -0700 Subject: [PATCH 019/660] selftests/bpf: Add inline-asm and subprog tests for R0: R2 returns Add inline-asm tests covering what the C tests cannot reach, since they need a callee that violates the convention on purpose. The coverage includes BPF-to-BPF returns, kfunc calls, backtracking and liveness. In addition, a negative freplace test checks that an extension cannot replace a function returning R0:R2. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055325.3300712-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- .../selftests/bpf/prog_tests/aggregate_ret.c | 11 + .../selftests/bpf/prog_tests/fexit_bpf2bpf.c | 16 ++ .../selftests/bpf/progs/aggregate_ret_func.c | 237 ++++++++++++++++++ .../selftests/bpf/progs/aggregate_ret_kfunc.c | 121 +++++++++ .../bpf/progs/aggregate_ret_target.c | 29 +++ .../bpf/progs/compute_live_registers.c | 28 +++ .../selftests/bpf/progs/freplace_ret_pair.c | 12 + .../selftests/bpf/test_kmods/bpf_testmod.c | 37 +++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 20 ++ 9 files changed, 511 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_ret.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_func.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_target.c create mode 100644 tools/testing/selftests/bpf/progs/freplace_ret_pair.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c new file mode 100644 index 000000000000..e0b94ed10f94 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "aggregate_ret_func.skel.h" +#include "aggregate_ret_kfunc.skel.h" + +void test_aggregate_ret(void) +{ + RUN_TESTS(aggregate_ret_func); + RUN_TESTS(aggregate_ret_kfunc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 2523c07a16c6..6262c81cdaa9 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -441,6 +441,20 @@ static void test_func_replace_int_with_void(void) " doesn't match type INT of global_func2()"); } +static void test_func_replace_ret_pair(void) +{ + const char *msg = "Return type of new_agg_ret_target_func() has size 8 " + "and of agg_ret_target_func() size 16"; + + /* + * An extension cannot replace a function whose return value comes back + * in the R0:R2 pair: only R0 is checked at the extension's exit, so it + * would leave R2 stale for the target's callers. + */ + test_obj_load_failure_common("freplace_ret_pair.bpf.o", + "./aggregate_ret_target.bpf.o", msg); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -660,6 +674,8 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_progmap(); if (test__start_subtest("freplace_int_with_void")) test_func_replace_int_with_void(); + if (test__start_subtest("freplace_ret_pair")) + test_func_replace_ret_pair(); if (test__start_subtest("freplace_void")) test_func_replace_void(); if (test__start_subtest("sleepable_fentry_to_xdp")) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c new file mode 100644 index 000000000000..6f66fc822ced --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +typedef unsigned __int128 u128; + +__naked u128 global_agg_good(void) +{ + asm volatile ( + "r0 = 0x1234;" /* low 64 bits */ + "r2 = 0x5678;" /* high 64 bits */ + "exit;" + ); +} + +__naked u128 global_agg_bad(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked u128 global_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_global_fail(void) +{ + asm volatile ( + "call %[global_agg_bad];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__failure __msg("At subprogram exit the register R2 is not a scalar value") +__naked int aggregate_ret_global_ptr_fail(void) +{ + asm volatile ( + "call %[global_agg_bad_ptr];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" /* stack pointer placed in the second return register */ + "exit;" + ); +} + +/* + * R2 is a return register once the subprogram returns a pair, so a stack + * pointer left in it is rejected at the callee's exit exactly as one in R0 + * is: the callee frame is gone by the time the caller could use it. + */ +SEC("tc") +__load_if_JITed() +__failure __msg("cannot return stack pointer to the caller") +__naked int aggregate_ret_static_ptr_fail(void) +{ + asm volatile ( + "call %[static_agg_bad_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_no_r2(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_static_uninit_fail(void) +{ + asm volatile ( + "call %[static_agg_no_r2];" + "r0 = r2;" + "exit;" + : + : __imm(static_agg_no_r2) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_precise(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 4;" /* second half; its value is made precise below */ + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r2 stack= before 11: (b7) r2 = 4") +__naked int aggregate_ret_static_precise(void) +{ + asm volatile ( + "call %[static_agg_precise];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_precise) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 0: (85) call pc+9") +__naked int aggregate_ret_global_precise(void) +{ + asm volatile ( + "call %[global_agg_good];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(global_agg_good) + : __clobber_all); +} + +#if defined(__clang_major__) && __clang_major__ >= 23 + +/* A by-value struct that smuggles a pointer, which must be rejected. */ +struct with_ptr { + void *p; + __u64 x; +}; + +/* A by-value union that smuggles a pointer, which must be rejected too. */ +union upair_with_ptr { + void *p; + __u64 halves[2]; +}; + +__naked struct with_ptr global_ret_struct_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_struct_ptr() has unsupported return type") +__naked int aggregate_ret_global_struct_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_struct_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_struct_ptr) + : __clobber_all); +} + +__naked union upair_with_ptr global_ret_union_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_union_ptr() has unsupported return type") +__naked int aggregate_ret_global_union_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_union_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_union_ptr) + : __clobber_all); +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c new file mode 100644 index 000000000000..d6b422ae9784 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Reference kfunc addresses to force those BTF to be emitted. Taking the address + * (rather than calling) avoids any dependence on the compiler lowering an + * __int128 or struct return value, which the BPF backend only supports from + * LLVM 23 on. + */ +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_i128), + "r"(&bpf_kfunc_call_test_ret_fastcall), + "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_big)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 7 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 6: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 5: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 2: (85) call bpf_kfunc_call_test_i128") +__naked int aggregate_ret_kfunc_precise(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_i128];" + "r6 = r2;" /* second return half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_i128) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("kfunc bpf_kfunc_call_test_ret_fastcall with >8-byte return is not supported with KF_FASTCALL") +__naked int aggregate_ret_kfunc_fastcall_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_ret_fastcall];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_fastcall) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars") +__naked int aggregate_ret_kfunc_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_kfunc_small_no_r2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "call %[bpf_kfunc_call_test_ret_ii];" + "r0 = r2;" /* R2 is not a return register for a <=8 byte struct */ + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ii) + : __clobber_all); +} + +/* + * A return value larger than 16 bytes does not fit in R0:R2 and is rejected by + * btf_distill_func_proto() before the KF_FASTCALL and JIT-capability checks. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("The function bpf_kfunc_call_test_ret_big return type STRUCT is unsupported") +__naked int aggregate_ret_kfunc_too_big_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_big];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_big) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_target.c b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c new file mode 100644 index 000000000000..cffd8d7d3241 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +/* freplace target: a global subprogram returning 16 bytes in R0:R2. */ +__naked unsigned __int128 agg_ret_target_func(void) +{ + asm volatile ( + "r0 = 0x1234;" + "r2 = 0x5678;" + "exit;" + ); +} + +SEC("tc") +__naked int agg_ret_target(void) +{ + asm volatile ( + "call %[agg_ret_target_func];" + "r0 = 0;" + "exit;" + : + : __imm(agg_ret_target_func) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index d055fc7b3b95..338e67cd8856 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -431,6 +431,34 @@ __naked void subprog1(void) ::: __clobber_all); } +static __used __naked unsigned __int128 aux2(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +/* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ +__load_if_JITed() +__log_level(2) +__msg("0: .12345.... (85) call pc+2") +__msg("1: ..2....... (bf) r0 = r2") +__msg("2: 0......... (95) exit") +__msg("3: .......... (b7) r0 = 1") +__msg("4: 0......... (b7) r2 = 2") +__msg("5: 0.2....... (95) exit") +__naked void subprog_ret_reg_pair(void) +{ + asm volatile ( + "call aux2;" + "r0 = r2;" + "exit;" + ::: __clobber_all); +} + #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/freplace_ret_pair.c b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c new file mode 100644 index 000000000000..12c15d293bd7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +SEC("freplace/agg_ret_target_func") +__u64 new_agg_ret_target_func(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index c4bc8e11c6e1..850cf4f830c4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -966,8 +966,41 @@ __bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) return r; } + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) +{ + struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) +{ + struct prog_test_ret_ii r = { .a = a, .b = b }; + + return r; +} #endif /* __x86_64__ || __aarch64__ */ +/* + * Takes no argument on purpose: with no arguments there is nothing for the sret + * pointer to displace, so this needs no architecture guard even though it + * returns 24 bytes. + */ +__bpf_kfunc struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) +{ + struct prog_test_ret_big r = { .a = 1, .b = 2, .c = 3 }; + + return r; +} + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1504,7 +1537,11 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test5) #if defined(__x86_64__) || defined(__aarch64__) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 426000f9a14f..65e693ada736 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -60,6 +60,22 @@ struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ __u64 hi; }; +struct prog_test_ret_ii { /* 8 bytes: R0 only */ + int a; + int b; +}; + +struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ + void *p; + __u64 tag; +}; + +struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ + __u64 a; + __u64 b; + __u64 c; +}; + struct prog_test_fail1 { void *p; int x; @@ -140,6 +156,10 @@ int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; __int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; #endif struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; +struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; +struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; From d50d9a06c86a76e800071d184d05c9cdbfa1ec4b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:30 -0700 Subject: [PATCH 020/660] Documentation/bpf: Document up to 16-byte kfunc return values in R0: R2 kfuncs may now return a value larger than 8 bytes and up to 16 bytes (a scalar-only struct or union, or an __int128), passed back in the R0:R2 register pair. Add a kfunc return-value section documenting this, including that a struct or union up to 8 bytes is returned in R0 alone, which struct and union members are accepted, that the R0:R2 register pair requires JIT support (bpf_jit_supports_kfunc_ret_reg_pair()), and that a return value larger than 16 bytes is unsupported. Also note that the same convention applies to BPF subprogram returns, and document the consequence for a global subprogram: it must assign both halves of a register-pair return, since an unassigned R2 may be left holding a pointer argument and is then rejected as a leak. Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055330.3301120-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- Documentation/bpf/kfuncs.rst | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 85f73e0bbd0f..89dea6b0b024 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -575,6 +575,71 @@ is also covered by this recovery. A kfunc handed an arena pointer may therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking against the arena. Larger accesses must verify the range explicitly. +2.9 kfunc Return Values +----------------------- + +A kfunc may return a scalar, a pointer, or a small struct or union by +value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. + +A struct or union returned by value must be composed only of scalars +(recursively), where a scalar is an integer or an enum; arrays of scalars are +allowed as members. Its bytes are handed back to the program as the raw +contents of R0 (and R2), so a pointer field would be laundered into a scalar +and escape the verifier's pointer provenance and reference tracking. A struct +or union with a pointer member is therefore rejected at load time, and so is +one with a floating-point member, which the ABI may not return in R0:R2 at +all. + +A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a +scalar-only struct or union, or an ``__int128``. Such a value is returned +in the register pair R0:R2, matching the convention LLVM uses for the BPF +target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or +union of 8 bytes or less is returned in R0 alone. + +:: + + struct bpf_pair { __u64 a, b; }; /* 16 bytes */ + + __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void) + { + struct bpf_pair p = { .a = 1, .b = 2 }; + + return p; /* p.a in R0, p.b in R2 */ + } + +Returning a value in the R0:R2 pair requires the JIT to place the second +half of the return value into R2, which not every architecture supports +right now. A kfunc with a return value larger than 8 bytes is therefore +rejected at load time on a JIT that does not advertise this capability (see +``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run +by the interpreter. A return value larger than 16 bytes is not supported. + +The same R0:R2 convention applies to a BPF subprogram, global or static, that +returns an ``__int128`` or a struct or union larger than 8 bytes. It is only +used when the program is JITed, since the interpreter propagates only R0 out of +a subprogram: without a JIT the return value stays in R0 alone, and a caller +reading R2 is rejected for reading an uninitialized register. A global +subprogram is verified in isolation, so its by-value struct or union return is +restricted to scalars just like a kfunc's; a static subprogram is verified +inline and has no such restriction. The main program is not covered: its return +value is the program's exit code, read out of R0 alone, so a declared upper +half is never looked at. + +A global subprogram must leave a scalar in *every* register of the pair, so +both halves of the returned value have to be assigned. Leaving the upper half +uninitialized is not merely untidy: the compiler is then free to leave R2 +holding whatever it happened to hold, which for a subprogram taking a pointer +argument is typically that pointer. Handing the caller an unknown scalar built +from a pointer is a leak, so the verifier rejects it with:: + + At subprogram exit the register R2 is not a scalar value (...) + +Initialize the whole return value, for example ``struct pair p = {};``, to +avoid this. A static subprogram is exempt from the scalar-only rule: it is +verified inline, so an unassigned R2 is simply passed back to the caller as +uninitialized and only a caller that reads it fails. A stack pointer left in +R2 is still rejected there, just as one in R0 is. + .. _BPF_kfunc_lifecycle_expectations: 3. kfunc lifecycle expectations From e40f61a31688898e7cf81e91074a42f805883e4d Mon Sep 17 00:00:00 2001 From: Darren Carreras Date: Thu, 13 Aug 2026 20:43:18 -0400 Subject: [PATCH 021/660] libbpf: Avoid overflow in BTF.ext bounds check A malformed BTF.ext subsection length in an ELF input can wrap the pointer addition used by btf_ext_parse_sec_info() on 32-bit builds. The wrapped pointer passes the bounds check and parsing then reads beyond the copied BTF.ext data. Ensure the header fits in the copied data, then validate the subsection offset and length with subtraction before forming its pointer. Fixes: ae4ab4b4117d ("btf: expose API to work with raw btf_ext data") Closes: https://issues.oss-fuzz.com/issues/477315119 Signed-off-by: Darren Carreras Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260813-b4-libbpf-v6-send-20260813-v6-1-56be61c52758@gmail.com --- tools/lib/bpf/btf.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 8417de92d028..c783359977b4 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -3364,7 +3364,7 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, { const struct btf_ext_info_sec *sinfo; struct btf_ext_info *ext_info; - __u32 info_left, record_size; + __u32 data_left, info_left, record_size; size_t sec_cnt = 0; void *info; @@ -3377,16 +3377,20 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, return -EINVAL; } - /* The start of the info sec (including the __u32 record_size). */ - info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; - info_left = ext_sec->len; + if (btf_ext->hdr->hdr_len > btf_ext->data_size) + return -EINVAL; - if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { + data_left = btf_ext->data_size - btf_ext->hdr->hdr_len; + if (ext_sec->off > data_left || ext_sec->len > data_left - ext_sec->off) { pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", ext_sec->desc, ext_sec->off, ext_sec->len); return -EINVAL; } + /* The start of the info sec (including the __u32 record_size). */ + info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; + info_left = ext_sec->len; + /* At least a record size */ if (info_left < sizeof(__u32)) { pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); From 669e4fa766000ae4137bb02eb22ce75ba78ad32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Tue, 18 Aug 2026 20:43:00 +0200 Subject: [PATCH 022/660] selftests/bpf: Allocate a larger timeout for connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some tests, like tc_tunnel or tc_edt, sporadically fail in CI with the following logs: (network_helpers.c:309: errno: Operation now in progress) \ Failed to connect to server send_and_test_data:FAIL:connect to server unexpected error: -115 This is due to SO_RCVTIMEO and SO_SNDTIMEO being set on the client socket (see settimeo() in client_socket()), allowing connect() to return an error and to set errno to EINPROGRESS instead of ETIMEDOUT. Increasing the timeout value for those tests is likely not a good solution (and it has already been done by commit 2790db208b44 ("selftests/bpf: Improve tc_tunnel test reliability")): some tests expect some data transfer to fail, and so the timeout value would increase overall test execution duration again (not only the connection, but any socket operation). Another solution is to allocate a timeout budget specific to the connection: we can apply a larger timeout only for connections, and once the connection is established, set back the timeout configured through opts->timeout_ms; this would allow connection to succeed under heavy CI load, while keeping timeout reasonable for the rest of the test traffic. Create a dedicated connect helper that sets a large timeout for the connect step, tries to connect, and if successful, finally sets the timeout value to the one configured by the test. Fixes: 99126abec5e5 ("bpf: selftests: A few improvements to network_helpers.c") Signed-off-by: Alexis Lothoré (eBPF Foundation) Signed-off-by: Andrii Nakryiko Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260818-tc_tunnel_flaky-v7-1-aec161e74ef2@bootlin.com --- tools/testing/selftests/bpf/network_helpers.c | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index cdf2d7d3ab32..d27dc79475de 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -49,6 +49,8 @@ errno = __save; \ }) +#define CONNECT_MIN_TIMEOUT_MS 5000 + struct ipv4_packet pkt_v4 = { .eth.h_proto = __bpf_constant_htons(ETH_P_IP), .iph.ihl = 5, @@ -291,6 +293,37 @@ error_close: return -1; } +static int connect_with_timeout(int fd, const struct sockaddr_storage *addr, + socklen_t addrlen, int timeout_ms) +{ + int connect_timeout_ms = MAX(timeout_ms, CONNECT_MIN_TIMEOUT_MS); + + /* + * Override timeout configuration with a larger value for the + * connection + */ + if (settimeo(fd, connect_timeout_ms)) { + log_err("Failed to set connect timeout"); + return -1; + } + + if (connect(fd, (const struct sockaddr *)addr, addrlen)) { + log_err("Failed to connect"); + return -1; + } + + /* + * If the timeout configured by the test is different from the + * connect timeout, restore it + */ + if (timeout_ms != connect_timeout_ms && settimeo(fd, timeout_ms)) { + log_err("Failed to set timeout for connected socket"); + return -1; + } + + return 0; +} + int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t addrlen, const struct network_helper_opts *opts) { @@ -305,8 +338,7 @@ int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t add return -1; } - if (connect(fd, (const struct sockaddr *)addr, addrlen)) { - log_err("Failed to connect to server"); + if (connect_with_timeout(fd, addr, addrlen, opts->timeout_ms)) { save_errno_close(fd); return -1; } @@ -376,20 +408,12 @@ int connect_fd_to_fd(int client_fd, int server_fd, int timeout_ms) struct sockaddr_storage addr; socklen_t len = sizeof(addr); - if (settimeo(client_fd, timeout_ms)) - return -1; - if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { log_err("Failed to get server addr"); return -1; } - if (connect(client_fd, (const struct sockaddr *)&addr, len)) { - log_err("Failed to connect to server"); - return -1; - } - - return 0; + return connect_with_timeout(client_fd, &addr, len, timeout_ms); } int make_sockaddr(int family, const char *addr_str, __u16 port, From bd2466ed8fc2ee54408bfe433e75f9f46bfa04f7 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:24 +0800 Subject: [PATCH 023/660] bpf: Roll back stream capacity when allocation fails bpf_stream_push_str() accounts the string length before allocating a stream element. If the allocation fails, the length remains charged even though no element is queued and therefore cannot be released by a reader. Repeated failures can exhaust the stream capacity permanently until the BPF program is freed. Refactor bpf_stream_release_capacity() to take a length so the consume and release sides are symmetric, and use it to roll back the charge when creating the stream element fails. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/c0ad7b18c59be6695fcc9e4e1200b2f4d5cbbea2.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98e9469..0b157ec4e38e 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -68,10 +68,8 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } @@ -79,7 +77,14 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l { int ret = bpf_stream_consume_capacity(stream, len); - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) @@ -188,7 +193,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } From effa6370ba89e7839c9bfeb802d8f05d6843ddf1 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:25 +0800 Subject: [PATCH 024/660] bpf: Fix stream capacity leak in staging path bpf_stream_stage_printk() increments ss->len before pushing the formatted string to the staging log. If element allocation fails, ss->len remains inflated and bpf_stream_stage_commit() permanently charges the stream capacity for data that was never queued. Only account the string length after a successful push, and use vscnprintf() so the staged length is the truncated payload without the trailing NUL. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/46898fb1ae482b2f8b1da459922b187378dfdc1d.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 0b157ec4e38e..a36fc9338967 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -316,17 +316,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + ret = __bpf_stream_push_str(&ss->log, buf->buf, len); + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } From 16790357a50baae10de1f619b608a85145d54bc2 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:26 +0800 Subject: [PATCH 025/660] bpf: Return partial progress from bpf_stream_read on fault bpf_stream_read() pops and frees stream elements after a successful copy_to_user(). If a later copy_to_user() fails, it currently restores only the current element's consumed_len and returns -EFAULT, hiding bytes already delivered to userspace and making the consumed data unrecoverable on retry. On a short copy, keep the successfully copied prefix of the current element and return the number of bytes copied. Return -EFAULT only when no bytes were copied for the call. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/7668c5838c6beafeea719df8d55319e2fe08cfef.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index a36fc9338967..09471a605268 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -167,6 +167,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) while (rem_len) { int pos = len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -180,13 +181,14 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; + chunk = elem->consumed_len - cons_len; - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (len == rem_len) ? -EFAULT : 0; break; } From 41c0348f6e65c26df4866d5dc96c57b1995fe36c Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:27 +0800 Subject: [PATCH 026/660] bpf: Reject oversized bpf_stream_vprintk output with -E2BIG bstr_printf() returns the would-be length excluding the trailing NUL. When that value is >= MAX_BPRINTF_BUF the message was truncated, but bpf_stream_push_str() still tried to allocate with the inflated length and failed with -ENOMEM. The boundary case of exactly MAX_BPRINTF_BUF could also copy the trailing NUL into the stream element. Reject such lengths with -E2BIG before charging stream capacity, and tighten bpf_stream_elem_alloc() to accept only payloads strictly shorter than the bprintf buffer. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/74387ae5024d7e2996095e97211430285095b525.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 09471a605268..7a5c3ac8676b 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -22,11 +22,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -245,6 +245,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); From 0cf194ccf78852377458b10c8c2e308678b10efa Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:28 +0800 Subject: [PATCH 027/660] selftests/bpf: Cover stream capacity and partial read edge cases Add coverage for the stream fixes requested on the capacity rollback series: - oversized bpf_stream_printk() returns -E2BIG and does not leak capacity for a subsequent successful write on the same program; - bpf_prog_stream_read() returns the successfully copied prefix when the userspace buffer straddles an unmapped page. Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/c5a79efbd3cb52cc0668ab50db5379c907cfe6a6.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../testing/selftests/bpf/prog_tests/stream.c | 63 +++++++++++++++++++ tools/testing/selftests/bpf/progs/stream.c | 28 +++++++++ 2 files changed, 91 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4caa60b4f9b 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -58,6 +58,69 @@ void test_stream_syscall(void) stream__destroy(skel); } +void test_stream_oversize(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_oversize); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "oversize run"); + ASSERT_OK(opts.retval, "oversize retval"); + + stream__destroy(skel); +} + +void test_stream_partial_read(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + long page_size; + char *page, *buf; + char rest[8] = {}; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "ret"); + ASSERT_OK(opts.retval, "retval"); + + page_size = sysconf(_SC_PAGESIZE); + page = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(page, MAP_FAILED, "mmap")) { + stream__destroy(skel); + return; + } + /* Leave only the first page mapped so a straddling copy faults. */ + if (!ASSERT_OK(munmap(page + page_size, page_size), "munmap second page")) { + munmap(page, page_size * 2); + stream__destroy(skel); + return; + } + + buf = page + page_size - 1; + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 3, NULL); + ASSERT_EQ(ret, 1, "partial bytes"); + ASSERT_EQ(buf[0], 'f', "first byte"); + + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, rest, sizeof(rest), NULL); + ASSERT_EQ(ret, 2, "remaining bytes"); + ASSERT_OK(memcmp(rest, "oo", 2), "remaining data"); + + munmap(page, page_size); + stream__destroy(skel); +} + static void test_address(struct bpf_program *prog, unsigned long *fault_addr_p) { LIBBPF_OPTS(bpf_test_run_opts, opts); diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 8d8e53d37266..3d9c0a63fab8 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -36,7 +36,12 @@ struct { } array SEC(".maps"); #define ENOSPC 28 +#define E2BIG 7 #define _STR "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define _X64 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +/* 1024 bytes: truncated by bstr_printf, must return -E2BIG. */ +#define _BIG_STR (_X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64 \ + _X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64) int size; u64 fault_addr; @@ -118,6 +123,29 @@ int stream_syscall(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int stream_oversize(void *ctx) +{ + int ret; + + ret = bpf_stream_printk(BPF_STDOUT, _BIG_STR); + if (ret != -E2BIG) + return ret ?: 1; + + /* The oversized output must not reduce the remaining stream capacity. */ + size = 0; + bpf_repeat(BPF_MAX_LOOPS) { + ret = bpf_stream_printk(BPF_STDOUT, _STR); + if (ret == -ENOSPC) + return size == 99954 ? 0 : 1; + if (ret) + return ret; + size += sizeof(_STR) - 1; + } + return 1; +} + SEC("syscall") __arch_x86_64 __arch_arm64 From 5e289c5a4a526e870efac0ca658635c47138593a Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Sat, 22 Aug 2026 14:55:33 +0800 Subject: [PATCH 028/660] bpf: Annotate bpf_obj_memcpy with data_race syzbot reported KCSAN write-write races when two tasks concurrently update the same map value. Both accesses reach the ordinary memcpy() paths in bpf_obj_memcpy() through copy_map_value(). Unlocked in-place updates of published map values are intentionally not serialized and may produce torn values. Callers requiring consistency must provide synchronization appropriate for the map type. bpf_long_memcpy() already annotates the same behavior for long-aligned copies. Annotate the ordinary memcpy() sites in bpf_obj_memcpy() with data_race(), matching bpf_long_memcpy(). This documents the existing concurrency semantics and suppresses KCSAN reports for these intentional races without changing synchronization or map update behavior. Reported-by: syzbot+44044637ef892e79ca2b@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=44044637ef892e79ca2b Signed-off-by: Quanye Yang Link: https://lore.kernel.org/bpf/20260822-bpf-kcsan-obj-memcpy-v3-1-4227b2edd8b4@proton.me Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ffa5626411ac..b3cd28d9e3b5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -572,7 +572,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +580,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) From b4757714cc63a18c3850dfe99b3f3d1b67b14bf9 Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Wed, 19 Aug 2026 20:09:23 +0800 Subject: [PATCH 029/660] bpf, riscv: Add support for signed arena loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed loads from arena memory are currently rejected on riscv64, as bpf_jit_supports_insn() refuses BPF_MEMSX loads when in_arena is set, while x86 and arm64 gained support for them in v6.18. Compilers such as GCC-14 are free to generate signed loads into arena memory, which breaks loading of otherwise valid BPF programs on riscv64. Implement BPF_PROBE_MEM32SX support in the RV64 JIT by reusing the existing arena handling: the arena base (RV_REG_ARENA) is added to the source register and the load is emitted with sign extension (lb/lh/lw). Add BPF_PROBE_MEM32SX to the add_exception_handler() mode gate so that faulting loads get an exception table entry which clears the destination register and resumes execution. Since BPF_PROBE_MEM32SX shares its mode value (0xc0) with BPF_ATOMIC, the gate accepts it only for LDX class instructions so that plain atomic instructions do not register exception table entries. Verified by running the arena LDSX selftests (arena_ldsx_disasm, arena_ldsx_exception, arena_ldsx_s8/s16/s32) and the full arena_atomics test suite on riscv64 QEMU, all passing. Signed-off-by: Chen Pei Tested-by: Pu Lehui Reviewed-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/67422a421852c3eed2269c6def7b5569e727617f.1787140923.git.cp0613@linux.alibaba.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 47c7bf431ba8..e7378be171a9 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -747,6 +747,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + !(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX && + BPF_CLASS(insn->code) == BPF_LDX) && BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return 0; @@ -1908,13 +1910,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + /* LDX | PROBE_MEM32SX: dst = *(signed size *)(src + RV_REG_ARENA + off) */ + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W: { bool sign_ext; sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || - BPF_MODE(insn->code) == BPF_PROBE_MEMSX; + BPF_MODE(insn->code) == BPF_PROBE_MEMSX || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) { emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); rs = RV_REG_T2; } @@ -2145,10 +2153,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (insn->imm == BPF_CMPXCHG) return rv_ext_enabled(ZACAS); break; - case BPF_LDX | BPF_MEMSX | BPF_B: - case BPF_LDX | BPF_MEMSX | BPF_H: - case BPF_LDX | BPF_MEMSX | BPF_W: - return false; } } From 8f2087a1b43bbc57231eca4b99415cc18fb3429a Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Wed, 19 Aug 2026 20:09:24 +0800 Subject: [PATCH 030/660] selftests/bpf: Enable arena LDSX tests for riscv64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the riscv64 JIT supports signed arena loads (BPF_PROBE_MEM32SX), enable the arena LDSX tests on riscv64: add JIT disassembly assertions for arena_ldsx_disasm (arena base in s7, add into t2, sign-extending lw/lh/lb loads) and run arena_ldsx_exception and arena_ldsx_s8/s16/s32 on riscv64. Signed-off-by: Chen Pei Tested-by: Pu Lehui Reviewed-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/1bab696ec36f196fa6ec74a0f056030b39730da7.1787140923.git.cp0613@linux.alibaba.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../testing/selftests/bpf/progs/verifier_ldsx.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 41340877dc9d..ed0a0f159bc1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -286,6 +286,19 @@ __jited("add x11, x0, x28") __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") +__arch_riscv64 +__jited("add t2, a5, s7") +__jited("lw s3, 0x10(t2)") +__jited("add t2, a5, s7") +__jited("lh s3, 0x18(t2)") +__jited("add t2, a5, s7") +__jited("lb s3, 0x20(t2)") +__jited("add t2, a0, s7") +__jited("lw s4, 0x10(t2)") +__jited("add t2, a0, s7") +__jited("lh s4, 0x18(t2)") +__jited("add t2, a0, s7") +__jited("lb s4, 0x20(t2)") __naked void arena_ldsx_disasm(void *ctx) { asm volatile ( @@ -317,6 +330,7 @@ __description("Arena LDSX Exception") __success __retval(0) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_exception(void *ctx) { asm volatile ( @@ -338,6 +352,7 @@ __description("Arena LDSX, S8") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s8(void *ctx) { asm volatile ( @@ -369,6 +384,7 @@ __description("Arena LDSX, S16") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s16(void *ctx) { asm volatile ( @@ -400,6 +416,7 @@ __description("Arena LDSX, S32") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s32(void *ctx) { asm volatile ( From ce36e38beb6863664994f5676122359c8d323203 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 24 Aug 2026 21:57:46 +0200 Subject: [PATCH 031/660] selftests/bpf: Resolve bpftool through bpftool_helpers in core_reloc run_btfgen() shells out to a hardcoded "./bpftool". That path only exists in the build tree, where DEFINE_TEST_RUNNER_RULES symlinks bpftool next to each runner binary. After "make install" it is not there: commit 0b236ac75d04 ("selftests/bpf: Fix make install target") dropped bpftool from TEST_GEN_PROGS_EXTENDED, which used to flatten it into $(INSTALL_PATH), and installs it under $(INSTALL_PATH)/tools/sbin/ instead, matching detect_bpftool_path(). Running test_progs from the install tree therefore fails all 64 core_reloc_btfgen subtests: run_core_reloc_tests:FAIL:run_btfgen unexpected error: 32512 (errno 95) sh: line 1: ./bpftool: No such file or directory 32512 is 127 << 8, i.e. the shell could not find the binary. Use run_bpftool_command() from bpftool_helpers.h, which locates bpftool via $BPFTOOL, ./tools/sbin/bpftool or ../tools/sbin/bpftool, the last of which also covers the test_progs flavors. bpftool_helpers.c is already in TRUNNER_EXTRA_SOURCES, so no build change is needed. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260824195746.3070450-1-memxor@gmail.com --- tools/testing/selftests/bpf/prog_tests/core_reloc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 08963c82f30b..80432c4656b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -7,6 +7,7 @@ #include #include #include +#include "bpftool_helpers.h" static int duration = 0; @@ -985,16 +986,15 @@ static size_t roundup_page(size_t sz) static int run_btfgen(const char *src_btf, const char *dst_btf, const char *objpath) { - char command[4096]; + char args[4096]; int n; - n = snprintf(command, sizeof(command), - "./bpftool gen min_core_btf %s %s %s", + n = snprintf(args, sizeof(args), "gen min_core_btf %s %s %s", src_btf, dst_btf, objpath); - if (n < 0 || n >= sizeof(command)) + if (n < 0 || n >= sizeof(args)) return -1; - return system(command); + return run_bpftool_command(args); } static void run_core_reloc_tests(bool use_btfgen) From ea0b60fae724aa1147d4a07eda5131d3a9a9977e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:00 +0200 Subject: [PATCH 032/660] bpf: Reject oversized stream read buffers BPF_PROG_STREAM_READ_BY_FD exposes stream_buf_len as a u32, but the internal stream reader accepts an int. Values above INT_MAX therefore become negative before bpf_stream_read(), producing negative consumption lengths. The generic usercopy size check prevents the oversized copy, but emits a warning and reports EFAULT for an unchecked argument. Change bpf_prog_stream_read() to accept u32 and reject values that the signed reader cannot represent. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-2-memxor@gmail.com --- include/linux/bpf.h | 2 +- kernel/bpf/stream.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b3cd28d9e3b5..c2027688be3e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4101,7 +4101,7 @@ void bpf_put_buffers(void); void bpf_prog_stream_init(struct bpf_prog *prog); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 7a5c3ac8676b..2b80a0599865 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -203,13 +203,15 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) return ret ? ret : len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } From 05ea1b6e2a5d008c24b7a2c6eaa3f708d9026994 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:01 +0200 Subject: [PATCH 033/660] selftests/bpf: Cover oversized stream read buffers Exercise BPF_PROG_STREAM_READ_BY_FD with a UINT_MAX buffer length and require EINVAL. Continue with the existing two-part read of the queued payload to verify that rejecting the oversized request does not consume or otherwise disturb stream data. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-3-memxor@gmail.com --- tools/testing/selftests/bpf/prog_tests/stream.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index 4d8054a23cc7..74bd15c4bfba 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -46,6 +46,10 @@ void test_stream_syscall(void) ret = -errno; ASSERT_EQ(ret, -EFAULT, "bad stream buf"); + ASSERT_LT(bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, UINT_MAX, NULL), 0, "error"); + ret = -errno; + ASSERT_EQ(ret, -EINVAL, "large stream buf"); + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); ASSERT_EQ(ret, 2, "bytes"); ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); From 8b94878b2572f0f41e6002d0fe2cc4086ddc2a47 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 25 Aug 2026 23:04:33 +0800 Subject: [PATCH 034/660] libbpf: Fix usdt attach failure when nop10 crosses page boundary The kernel refuses to attach to a nop10 that crosses a page boundary, since it can't be atomically rewritten: /* can_optimize(), arch/x86/kernel/uprobes.c */ /* We can't do cross page atomic writes yet. */ return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; Whether the nop10 crosses a page is purely up to the binary layout, so this does happen in practice. libbpf doesn't check for it and blindly shifts the uprobe onto the nop10, and the attach then fails with -ENOTSUPP. Just keep the uprobe on the preceding 1-byte nop in that case, it works everywhere as a regular int3 uprobe. Fixes: ee2862439e5c ("libbpf: Change has_nop_combo to work on top of nop10") Signed-off-by: Jiayuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260825150444.31603-1-jiayuan.chen@linux.dev --- tools/lib/bpf/usdt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 2e56e3ab5b6c..ee9d1b614883 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -608,8 +608,12 @@ static bool has_nop_combo(int fd, long off) unsigned char nop_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; + long page_sz = getpagesize(); unsigned char buf[11]; + /* the kernel can't attach to a nop10 that crosses a page boundary */ + if ((off + 1) % page_sz + 10 > page_sz) + return false; if (pread(fd, buf, 11, off) != 11) return false; return memcmp(buf, nop_combo, 11) == 0; From 1555de33018fad06ca5dfa2fad5d3d1c6a6eb2fe Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 25 Aug 2026 23:04:34 +0800 Subject: [PATCH 035/660] selftests/bpf: Add test for usdt probe with page-crossing nop10 Add a USDT probe laid out so that its nop10 crosses a page boundary and check that attaching to it succeeds and the probe fires. Without the previous fix libbpf shifts the uprobe onto the nop10 and the attach fails with -ENOTSUPP. Signed-off-by: Jiayuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260825150444.31603-2-jiayuan.chen@linux.dev --- tools/testing/selftests/bpf/prog_tests/usdt.c | 54 +++++++++++++++++++ tools/testing/selftests/bpf/usdt_2.c | 16 ++++++ 2 files changed, 70 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 8004c9568ffa..eff1e57ab13c 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,6 +250,7 @@ cleanup: #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_2_cross_page(void); extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; @@ -342,6 +343,57 @@ cleanup: test_usdt__destroy(skel); } +/* + * Test attachment to a USDT probe whose nop10 crosses a page boundary. + * The kernel can't optimize such nop10, so libbpf keeps the uprobe on + * the preceding 1-byte nop. Do not assume any particular placement + * here, though: however the probe ends up attached, the attachment + * must succeed and the probe must fire. + */ +static void subtest_optimized_attach_cross_page(void) +{ + long page_sz = sysconf(_SC_PAGESIZE); + struct test_usdt *skel; + __u8 *addr = NULL; + long i; + + /* combo is placed up to a page of padding after the function start */ + for (i = 0; i < 2 * page_sz; i++) { + if (!memcmp((void *)usdt_2_cross_page + i, nop1_nop10_combo, 11)) { + addr = (void *)usdt_2_cross_page + i; + break; + } + } + if (!ASSERT_OK_PTR(addr, "find_nop1_nop10_combo")) + return; + + /* layout sanity check: the nop10 must cross the page boundary */ + if (!ASSERT_GT((unsigned long)(addr + 1) % page_sz + 10, page_sz, + "nop10_crosses_page")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->my_pid = getpid(); + + skel->links.usdt0 = bpf_program__attach_usdt(skel->progs.usdt0, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", + "usdt_2_cross_page", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt0, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2_cross_page(); + usdt_2_cross_page(); + + ASSERT_EQ(skel->bss->usdt0_called, 2, "usdt0_called"); + +cleanup: + test_usdt__destroy(skel); +} + /* * Test that USDT arguments survive nop10 optimization in a function where * the compiler places operands in the red zone. @@ -660,6 +712,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_attach_cross_page")) + subtest_optimized_attach_cross_page(); if (test__start_subtest("optimized_red_zone")) subtest_optimized_red_zone(); #endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 5e38f8605b02..3b7024b9b08b 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -13,6 +13,22 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +/* + * Force the nop1,nop10 combo of the USDT probe to a spot where the nop10 + * crosses a page boundary: .balign starts the padding exactly at a page + * start regardless of the compiler-generated prologue size, and the 4086 + * one-byte nops put the nop1 at page offset 4086, so the following nop10 + * occupies the last 9 bytes of that page and 1 byte of the next one. + * The kernel can't optimize such nop10, so libbpf must keep the uprobe + * on the 1-byte nop. + */ +__attribute__((noinline)) +void usdt_2_cross_page(void) +{ + asm volatile (".balign 4096, 0x90\n\t.skip 4086, 0x90"); + USDT(optimized_attach, usdt_2_cross_page); +} + static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; From b0eeb9e0f548ab4b555b588a8cbb09de7a690c47 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:40 -0700 Subject: [PATCH 036/660] selftests/bpf: Name the remaining placeholder programs dummy_test [NFC] Thirteen of the fifteen feature-gated placeholder programs are called dummy_test. Two are not: hence this patch. This is purely cosmetic as nothing depends on those names. The non-compliant setups use RUN_TESTS(skel) which goes through all programs normally. This is just more consistent and grep friendly. Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-2-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/arena_kfunc.c | 2 +- tools/testing/selftests/bpf/progs/stack_arg_fail.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index bf0d304e0e59..47bed5c4c488 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -234,7 +234,7 @@ __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") __success -int arena_arg_stack(void *ctx) +int dummy_test(void *ctx) { return 0; } diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index ad9d4bfe15dc..6f6f59306d0e 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -104,7 +104,7 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") __success -int test_stack_arg_big(struct __sk_buff *skb) +int dummy_test(struct __sk_buff *skb) { return 0; } From 8928f5ffdb65f0f69babb006123d256bd774f2d3 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:41 -0700 Subject: [PATCH 037/660] selftests/bpf: Report placeholder tests as SKIP, not OK Several verifier test files wrap their contents in a feature gate and provide a one-line placeholder program in the #else arm, so the file still yields something when the toolchain or JIT cannot support the tested instructions. However the placeholder is annotated __success, so it reports as a pass, whereas this is really a SKIP. | #318/1 verifier_ldsx/cpuv4 is not supported by compiler or jit, ...:OK | #318 verifier_ldsx:OK This causes skipped coverage to be overlooked: see the next patch where BPF_GCC builds miss a whole bunch of test due to clang only gate. Add a __skip("reason") annotation for a compile-time gate, and use it for the placeholders. After this change, above tests reported as | #318/1 verifier_ldsx/cpuv4 is not supported by compiler or jit, ...:SKIP | #318 verifier_ldsx:SKIP Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-3-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/arena_kfunc.c | 1 + tools/testing/selftests/bpf/progs/bpf_misc.h | 6 ++++++ tools/testing/selftests/bpf/progs/stack_arg_fail.c | 1 + tools/testing/selftests/bpf/progs/stack_arg_precision.c | 1 + .../testing/selftests/bpf/progs/verifier_aggregate_ret.c | 1 + tools/testing/selftests/bpf/progs/verifier_bswap.c | 1 + tools/testing/selftests/bpf/progs/verifier_gotol.c | 1 + tools/testing/selftests/bpf/progs/verifier_ldsx.c | 1 + .../testing/selftests/bpf/progs/verifier_load_acquire.c | 1 + tools/testing/selftests/bpf/progs/verifier_movsx.c | 1 + tools/testing/selftests/bpf/progs/verifier_percpu_addr.c | 1 + .../testing/selftests/bpf/progs/verifier_private_stack.c | 1 + tools/testing/selftests/bpf/progs/verifier_sdiv.c | 1 + tools/testing/selftests/bpf/progs/verifier_stack_arg.c | 1 + .../selftests/bpf/progs/verifier_stack_arg_order.c | 1 + .../testing/selftests/bpf/progs/verifier_store_release.c | 1 + tools/testing/selftests/bpf/test_loader.c | 9 +++++++++ 17 files changed, 30 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 47bed5c4c488..50609f3b0564 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -233,6 +233,7 @@ SEC("syscall") __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") +__skip("arena_arg_stack: not supported") __success int dummy_test(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 5eacf1b43252..88b0bfba83bb 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -106,6 +106,11 @@ * __description Text to be used for display and as an additional filter * alias, while the original program name stays matchable. * + * __skip Report the test as SKIP with the given reason instead of + * running it. For placeholder programs that stand in for a + * feature the toolchain or JIT cannot provide, so a run does + * not report OK for coverage it never executed. + * * __log_level Log level to use for the program, numeric value expected. * * __flag Adds one flag use for the program, the following values are valid: @@ -139,6 +144,7 @@ #define __failure __test_tag("test_expect_failure") #define __success __test_tag("test_expect_success") #define __description(desc) __test_tag("test_description=" desc) +#define __skip(reason) __test_tag("test_skip=" reason) #define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) #define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) #define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index 6f6f59306d0e..eed97d582515 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -103,6 +103,7 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") +__skip("stack_arg_fail: not supported") __success int dummy_test(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index bee2eeec021d..ce0301a41fa9 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -124,6 +124,7 @@ __naked void stack_arg_precision_bpf2bpf(void) SEC("socket") __description("stack_arg_precision: not supported, dummy test") +__skip("stack_arg_precision: not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c index 7851bade2b40..f5c1166027a1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -167,6 +167,7 @@ int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) SEC("socket") __description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__skip("needs LLVM 23") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index cffaf36192bc..074faef5480e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c @@ -117,6 +117,7 @@ l0_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_gotol.c b/tools/testing/selftests/bpf/progs/verifier_gotol.c index d5d8f24df394..c0fd30a3a4c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotol.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotol.c @@ -52,6 +52,7 @@ __naked void gotol_large_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index ed0a0f159bc1..5c704817111f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -453,6 +453,7 @@ void kfunc_root(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index d17026d7480d..bcc34e798c63 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -267,6 +267,7 @@ __naked void load_acquire_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index a4d8814eb5ed..5559c0583dd1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -343,6 +343,7 @@ label_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c index 967f4e6e3a49..fd418f27996f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -61,6 +61,7 @@ __naked void percpu_addr(void) SEC("raw_tp") __description("percpu addr dummy") +__skip("percpu addr tests need x86") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index ea0a7e73331d..3d9994a52464 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -377,6 +377,7 @@ int private_stack_max_depth(void) SEC("kprobe") __description("private stack is not supported, use a dummy test") +__skip("private stack is not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 95f3239ce228..515644ace516 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -1271,6 +1271,7 @@ __naked void smod32_int_min_mod_neg2_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 7e0ce5db28a0..51d22faf4559 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -436,6 +436,7 @@ __naked void stack_arg_sequential_calls(void) SEC("socket") __description("stack_arg is not supported by compiler or jit, use a dummy test") +__skip("stack_arg is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index c9fe4857da3f..8e4325273b3d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -174,6 +174,7 @@ __naked void stack_arg_read_without_write_2(void) SEC("socket") __description("stack_arg order is not supported by compiler or jit, use a dummy test") +__skip("stack_arg order is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_store_release.c b/tools/testing/selftests/bpf/progs/verifier_store_release.c index 72f1eb006074..0abbee5ce109 100644 --- a/tools/testing/selftests/bpf/progs/verifier_store_release.c +++ b/tools/testing/selftests/bpf/progs/verifier_store_release.c @@ -290,6 +290,7 @@ __naked void store_release_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 07807757b518..794a7dfb0579 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -70,6 +70,7 @@ struct test_spec { int arch_mask; int load_mask; int linear_sz; + const char *skip_reason; bool auxiliary; bool valid; }; @@ -456,6 +457,8 @@ static int parse_test_spec(struct test_loader *tester, continue; if ((val = str_has_pfx(s, "test_description="))) { description = val; + } else if ((val = str_has_pfx(s, "test_skip="))) { + spec->skip_reason = val; } else if (strcmp(s, "test_expect_failure") == 0) { spec->priv.expect_failure = true; spec->mode_mask |= PRIV; @@ -1327,6 +1330,12 @@ void run_subtest(struct test_loader *tester, if (!test__start_subtest_with_desc(subspec->name, subspec->description)) return; + if (spec->skip_reason) { + printf("%s:SKIP: %s\n", __func__, spec->skip_reason); + test__skip(); + return; + } + if ((get_current_arch() & spec->arch_mask) == 0) { test__skip(); return; From 48b69cc8825d411a4c4df32de854434e9235a1b0 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:42 -0700 Subject: [PATCH 038/660] selftests/bpf: Enable some of the blocked cpuv4 instruction tests for bpf-gcc A bunch of tests are gated behind clang only __clang_major__ and thus fail to build under BPF_GCC. What's worse is, before the prev patch they were also declared as passing/OK. This need not be because (a) bpf-gcc does support the feature in codegen and (b) also advertises correctly with feature support preprocessor macros: __BPF_FEATURE_BSWAP, __BPF_FEATURE_GOTOL, __BPF_FEATURE_LDSX, __BPF_FEATURE_MOVSX, __BPF_FEATURE_SDIV_SMOD, __BPF_FEATURE_ST So loosen up the gates to also add specific feature check. clang >=18 has to stay, despite it being true for so long: clang only defines __BPF_FEATURE_MOVSX and friends at -mcpu=v4, but assembles the inline asm at -mcpu=v3 too, and dropping it would remove these tests from the default test_progs flavour where they run today. Since bpf-gcc doesn't support addr_space_cast, wrap the corresponding tests under __BPF_FEATURE_ADDR_SPACE_CAST to skip under it. And also move the map under that guard so that a bpf-gcc object does not declare an arena map that no program in it references. Recovered under BPF_GCC (programs per object, was 1 dummy each): verifier_sdiv 80 verifier_movsx 17 verifier_ldsx 14 (20 under clang; 5 arena tests, kfunc_root and the arena map stay clang-only) verifier_bswap 13 CAN_USE_GOTOL and CAN_USE_BPF_ST only gate parts of a file, so those were not dummies to begin with: compute_live_registers 17 -> 19 (21 under clang) verifier_gotol 1 -> 2 verifier_iterating_callbacks 34 -> 35 The clang builds are unchanged: verifier_ldsx still emits 20 programs under the cpuv4 flavour. One thing left alone: verifier_load_acquire.c and verifier_store_release.c stay clang-only, and their gates are correct as gas doesn't implement those (experimental -mxbpf not implemented at all). FWIW LLVM groups those under -mcpu=v4 and defines __BPF_FEATURE_LOAD_ACQ_STORE_REL there. Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-4-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/bpf_misc.h | 4 ++-- .../bpf/progs/compute_live_registers.c | 8 ++++++++ .../selftests/bpf/progs/verifier_bswap.c | 2 +- .../selftests/bpf/progs/verifier_ldsx.c | 18 +++++++++++------- .../selftests/bpf/progs/verifier_movsx.c | 2 +- .../selftests/bpf/progs/verifier_sdiv.c | 2 +- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 88b0bfba83bb..eb88d9ce6c34 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -261,11 +261,11 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_GOTOL)) #define CAN_USE_GOTOL #endif -#if __clang_major__ >= 18 +#if __clang_major__ >= 18 || defined(__BPF_FEATURE_ST) #define CAN_USE_BPF_ST #endif diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index 338e67cd8856..941b65fe7c07 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -299,7 +299,15 @@ __naked void gotol(void) "r3 = 24;" "if r1 > 0x7 goto +2;" "r0 = r2;" +#ifdef __clang__ "gotol +1;" +#else + /* + * gas mis-parses 'gotol +1' as 'goto l+1', same encoding + * without the sign (binutils PR gas/34558). + */ + "gotol 1;" +#endif "r0 = r3;" "exit;" : diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index 074faef5480e..48ffb5b242d2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_BSWAP)) SEC("socket") __description("BSWAP, 16") diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 5c704817111f..7e25e1b8cd0b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -9,13 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 1); -} arena SEC(".maps"); + (__clang_major__ >= 18 || defined(__BPF_FEATURE_LDSX)) SEC("socket") __description("LDSX, S8") @@ -263,6 +257,14 @@ __naked void ldsx_ctx_8(void) : __clobber_all); } +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + SEC("syscall") __description("Arena LDSX Disasm") __success @@ -449,6 +451,8 @@ void kfunc_root(void) bpf_arena_alloc_pages(0, 0, 0, 0, 0); } +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index 5559c0583dd1..195b27a51224 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_MOVSX)) SEC("socket") __description("MOV32SX, S8") diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 515644ace516..d6cbe1e27fd6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -9,7 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_SDIV_SMOD)) SEC("socket") __description("SDIV32, non-zero imm divisor, check 1") From 713c02b59e21840f623a3e31c43d967026c75181 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 24 Aug 2026 17:26:56 +0800 Subject: [PATCH 039/660] bpftool: Fix spurious batch file read error do_batch() checks errno after the read loop to detect read failures, but fgets() does not clear errno on success, so a stale errno left by a previously executed command (e.g. map dump's EBADF from a double close) makes bpftool report a batch file read failure and exit with an error even though every command succeeded. Clear errno before each fgets() call, so the post-loop check only sees the outcome of the last read: zero on success or EOF, E2BIG for an overlong line, and a genuine errno when fgets() fails. Since errno is now reset before every read in batch mode, drop the USE_LIBCAP errno reset in main() that existed only to keep errno clean for the batch mode. Fixes: 71bb428fe2c1 ("tools: bpf: add bpftool") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260824092657.1789956-2-chenyuan_fl@163.com --- tools/bpf/bpftool/main.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index c91e1a6e1a1e..7a0c214f08a0 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -365,7 +365,11 @@ static int do_batch(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); - while (fgets(buf, sizeof(buf), fp)) { + for (;;) { + errno = 0; + if (!fgets(buf, sizeof(buf), fp)) + break; + cp = strchr(buf, '#'); if (cp) *cp = '\0'; @@ -467,16 +471,6 @@ int main(int argc, char **argv) setlinebuf(stdout); -#ifdef USE_LIBCAP - /* Libcap < 2.63 hooks before main() to compute the number of - * capabilities of the running kernel, and doing so it calls prctl() - * which may fail and set errno to non-zero. - * Let's reset errno to make sure this does not interfere with the - * batch mode. - */ - errno = 0; -#endif - last_do_help = do_help; pretty_output = false; json_output = false; From ac3d88577cdaa5330cf32d03ee3808df8fa338ac Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 24 Aug 2026 17:26:57 +0800 Subject: [PATCH 040/660] bpftool: Fix bypass of the batch line length check by comments do_batch() strips trailing comments by truncating the line at '#' before checking whether fgets() filled the buffer. If a batch line longer than the buffer contains a '#' within the first sizeof(buf) - 1 bytes, the truncation makes strlen(buf) smaller and the line-length check is bypassed. The unread remainder of the line then stays in the file stream and is parsed and executed as a separate command on the next loop iteration. Continuation lines handled below are affected the same way: an overlong continuation line containing '#' bypasses the "command is too long" check, and its unread remainder is executed as a separate command. Move the line-length checks before the comment is stripped, so they see the full line as read from the file and overlong lines are rejected regardless of comments. A line that fills the buffer exactly is now rejected as well, which is fine: batch command lines are not expected to come anywhere near the buffer limit. Fixes: 71bb428fe2c1 ("tools: bpf: add bpftool") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260824092657.1789956-3-chenyuan_fl@163.com --- tools/bpf/bpftool/main.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 7a0c214f08a0..5ababd8f7d0a 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -370,15 +370,15 @@ static int do_batch(int argc, char **argv) if (!fgets(buf, sizeof(buf), fp)) break; - cp = strchr(buf, '#'); - if (cp) - *cp = '\0'; - if (strlen(buf) == sizeof(buf) - 1) { errno = E2BIG; break; } + cp = strchr(buf, '#'); + if (cp) + *cp = '\0'; + /* Append continuation lines if any (coming after a line ending * with '\' in the batch file). */ @@ -391,15 +391,15 @@ static int do_batch(int argc, char **argv) goto err_close; } - cp = strchr(contline, '#'); - if (cp) - *cp = '\0'; - if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) { p_err("command %u is too long", lines); err = -1; goto err_close; } + + cp = strchr(contline, '#'); + if (cp) + *cp = '\0'; buf[strlen(buf) - 2] = '\0'; strcat(buf, contline); } From 5e875ae9e9ffd5c76c6bdf8982f99de340dd46af Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 26 Aug 2026 22:56:13 +0800 Subject: [PATCH 041/660] selftests/bpf: Avoid flaky resize value test for percpu data The bpf_map__set_value_size() test for percpu data relies on the tail 'cpu_id' field in the ".percpu" map. The test would fail, when 'cpu_id' is not the last field in the map. To avoid the flaky test, factor out a subtest to use a dedicated ".percpu.arr" map that only has one field 'int arr[1]'. And, skip the subtest when fail to load skel with errno E2BIG and the new value size is larger than ALIGN(32KiB, PAGE_SIZE). And, drop 'args' in test_percpu_data_on_cpus(), which isn't used in bpf prog side. And, enhance the test to cover 'set' and 'nums[6]'. Fixes: 4c9241bd731a ("selftests/bpf: Add tests to verify global percpu data") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260826145613.34182-1-leon.hwang@linux.dev --- .../bpf/prog_tests/global_data_init.c | 96 ++++++++++++++----- .../bpf/progs/test_global_percpu_data.c | 10 +- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 5671c31085cd..7c539cbcf3a1 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -68,12 +68,9 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f { struct test_global_percpu_data__percpu *data = NULL; int i, err, key = 0, num_online, run = 0; - __u64 args[2] = {0x1234ULL, 0x5678ULL}; size_t data_sz; bool *online; LIBBPF_OPTS(bpf_test_run_opts, topts, - .ctx_in = args, - .ctx_size_in = sizeof(args), .flags = BPF_F_TEST_RUN_ON_CPU, ); @@ -81,7 +78,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f if (!ASSERT_OK(err, "parse_cpu_mask_file")) return; - data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data_sz = sizeof(*data); data = calloc(1, data_sz); if (!ASSERT_OK_PTR(data, "calloc percpu data")) goto out; @@ -109,7 +106,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f break; ASSERT_EQ(*runp, ++run, "run"); - ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->cpu_id, i, "cpu_id"); ASSERT_EQ(data->data, 1, "data"); ASSERT_TRUE(data->set, "set"); ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); @@ -127,7 +124,6 @@ static void test_global_percpu_data_init(void) { struct test_global_percpu_data__percpu init_value = {}; struct test_global_percpu_data__percpu *init_data; - const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); struct test_global_percpu_data *skel = NULL; size_t init_data_sz; struct bpf_map *map; @@ -163,9 +159,11 @@ static void test_global_percpu_data_init(void) if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) goto out; + init_value.set = 1; init_value.data = 2; init_value.nums[6] = -1; init_value.struct_data.i = 2; + init_value.struct_data.set = 1; init_value.struct_data.nums[6] = -1; err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); if (!ASSERT_OK(err, "bpf_map__set_initial_value")) @@ -177,35 +175,24 @@ static void test_global_percpu_data_init(void) ASSERT_EQ(init_data->data, init_value.data, "init_value data"); ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->nums[6], init_value.nums[6], "init_value nums[6]"); ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.set, init_value.struct_data.set, + "init_value struct_data.set"); ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], "init_value struct_data.nums[6]"); ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], init_value.nums[6], "skel->percpu->nums[6]"); ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.set, init_value.struct_data.set, + "skel->percpu->struct_data.set"); ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], "skel->percpu->struct_data.nums[6]"); - ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); - err = bpf_map__set_value_size(map, desired_sz); - if (!ASSERT_OK(err, "bpf_map__set_value_size")) - goto out; - if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) - goto out; - if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) - goto out; - - init_data = bpf_map__initial_value(map, &init_data_sz); - if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) - goto out; - if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) - goto out; - if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) - goto out; - err = test_global_percpu_data__load(skel); if (!ASSERT_OK(err, "test_global_percpu_data__load")) goto out; @@ -236,6 +223,67 @@ out: test_global_percpu_data_lskel__destroy(lskel); } +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +static void test_global_percpu_data_map_resize(void) +{ + const __u32 page_size = sysconf(_SC_PAGE_SIZE); + const __u32 pcpu_max_value_size = ALIGN(PCPU_MIN_UNIT_SIZE, page_size); + const __u32 desired_sz = page_size * 2; + struct test_global_percpu_data *skel; + const int arr_value = -1; + struct bpf_map *map; + int err, prog_fd; + size_t data_sz; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + map = skel->maps.percpu_arr; + skel->percpu_arr->arr[0] = arr_value; + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "bpf_map__value_size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "bpf_map__btf_value_type_id")) + goto out; + + skel->percpu_arr = bpf_map__initial_value(map, &data_sz); + if (!ASSERT_EQ(data_sz, desired_sz, "data_sz")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_arr, "skel->percpu_arr")) + goto out; + if (!ASSERT_EQ(skel->percpu_arr->arr[0], arr_value, "arr[0]")) + goto out; + + err = test_global_percpu_data__load(skel); + if (bpf_map__value_size(map) > pcpu_max_value_size) { + if (ASSERT_EQ(err, -E2BIG, "test_global_percpu_data__load E2BIG")) + /* + * When the PAGE_SIZE is 64KiB, the new value size will + * be 128KiB. Creating a percpu_array map with the new + * value size will fail with -E2BIG due to percpu_array + * map value size limit 64KiB, which is calculated by + * PFN_ALIGN(32KiB). + * Let's skip the test for such case. + */ + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + map = skel->maps.percpu; + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + ASSERT_EQ(skel->bss->arr_sum, arr_value * skel->bss->run, "arr_sum"); + +out: + test_global_percpu_data__destroy(skel); +} + static int create_rdonly_percpu_array(void) { LIBBPF_OPTS(bpf_map_create_opts, map_opts, @@ -388,6 +436,8 @@ void test_global_percpu_data(void) test_global_percpu_data_init(); if (test__start_subtest("lskel")) test_global_percpu_data_lskel(); + if (test__start_subtest("map_resize")) + test_global_percpu_data_map_resize(); if (test__start_subtest("rdonly_direct_read")) test_global_percpu_data_rdonly_direct_read(); if (test__start_subtest("rdonly_direct_write")) diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 5dc21b3b4cb5..2765cd46e9af 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -8,9 +8,12 @@ int loong SEC(".percpu.looooooooong"); int data3 SEC(".data.percpu"); int data2 SEC(".percpu.data"); +/* Used for testing bpf_map__set_value_size(). */ +int arr[1] SEC(".percpu.arr"); +int arr_sum; + int run; -/* cpu_id as array to verify map value resizing. */ -int cpu_id[1] SEC(".percpu"); +int cpu_id SEC(".percpu"); int data SEC(".percpu") = -1; int nums[7] SEC(".percpu"); bool set SEC(".percpu") = false; @@ -34,7 +37,8 @@ int update_percpu_data(void *ctx) data = 1; run++; set = true; - cpu_id[0] = bpf_get_smp_processor_id(); + cpu_id = bpf_get_smp_processor_id(); + arr_sum += arr[0]; return 0; } From cd35e1b10182c42b4ae31ee49119463b17d8ba7f Mon Sep 17 00:00:00 2001 From: Tw Date: Fri, 28 Aug 2026 08:34:10 +0800 Subject: [PATCH 042/660] libbpf: Fix for the potential undefined behavior due to shifting When compiling libbpf with sanitation, ubsan will report the following: ``` left shift of 1 by 31 places cannot be represented in type 'int' ``` This may lead to undefined behavior according to the compiler implementation, let's fix it by casting to unsigned counterpart before shifting. Signed-off-by: Tan Wei Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260828003448.1684064-1-tw19881113@gmail.com --- tools/lib/bpf/libbpf.c | 2 +- tools/lib/bpf/libbpf_internal.h | 4 ++-- tools/testing/selftests/bpf/test_btf.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index b749c01742ee..f09cbfd8e729 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -11738,7 +11738,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, errstr(bit)); return bit; } - attr.config |= 1 << bit; + attr.config |= 1ULL << bit; } attr.size = attr_sz; attr.type = type; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4c46d34fc055..cb4d96233844 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -77,7 +77,7 @@ #define JUMPTABLES_SEC ".jumptables" #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) @@ -259,7 +259,7 @@ static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) static inline __u32 btf_type_info(int kind, int vlen, int kflag) { - return (kflag << 31) | (kind << 24) | vlen; + return ((__u32)kflag << 31) | (kind << 24) | vlen; } enum map_def_parts { diff --git a/tools/testing/selftests/bpf/test_btf.h b/tools/testing/selftests/bpf/test_btf.h index e65889ab4adf..e7bc78108374 100644 --- a/tools/testing/selftests/bpf/test_btf.h +++ b/tools/testing/selftests/bpf/test_btf.h @@ -7,7 +7,7 @@ #define BTF_END_RAW 0xdeadbeef #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) \ (name), (info), (size_or_type) From f62a9b82700c718cdf619fb36aaab31015ff87eb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:19 -0700 Subject: [PATCH 043/660] bpf: Record each half of a paired return value in verifier diagnostics A subprogram returning more than 8 bytes comes back in the R0:R2 register pair. There are three places, check_func_call(), prepare_func_exit() and check_kfunc_call(), where only R0 is tracked by the diagnostic modification scope and R2 is missed. Fix it by opening a diagnostic modification scope for each return register. This way, both return registers are recorded. Fixes: 0630ad00d96d ("bpf: Add verifier support for 16-byte returns in R0: R2") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061519.1692270-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 56 ++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b..5d8162e13c20 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9995,9 +9995,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, */ if (!returns_void) { nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; - for (i = 0; i < nregs; i++) - mark_reg_unknown(env, caller->regs, ret_regs[i]); + mark_reg_unknown(env, caller->regs, ret_regs[0]); bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } } if (env->subprog_info[subprog].might_throw) { @@ -10403,10 +10408,14 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * return to the caller whatever the callee had in the * return register(s) */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - for (i = 0; i < nregs; i++) - caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; - bpf_diag_mod_end(env); + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -11342,15 +11351,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* * Mark the register(s) holding a @size byte kfunc return value as unknown - * scalars. Both halves of a register pair are treated the same way. + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. */ -static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, u32 size) +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) { u32 i, nregs = ret_regs_cnt(size); - for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; } static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) @@ -13767,7 +13794,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, nargs, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14021,7 +14048,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); @@ -14039,7 +14066,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, btf_name_by_offset(desc_btf, t->name_off)); return -EINVAL; } - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14170,7 +14197,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); From 77983371742e37ff3117acdc5dfb1969cedd415c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:24 -0700 Subject: [PATCH 044/660] bpf: Drop the recursion depth argument of btf_type_is_scalar_struct() btf_type_is_scalar_struct() recurses into nested struct members and carries the nesting depth in a @rec argument, so every caller has to spell out the 0 that starts the walk. Move the recursion into btf_struct_member_walk() and leave btf_type_is_scalar_struct() as a thin wrapper over it. This is groundwork for the later patches, which give the walk further state that callers should not have to supply: the kinds of member it accepts, and a record of the member that made it fail. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061524.1692957-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 24 +++++++++++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521..3eb61edc8c5e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1489,7 +1489,7 @@ struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699..47d43eb983a5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7995,7 +7995,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (local_func || btf_type_is_scalar_struct(env, btf, t)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d8162e13c20..ed077a5422af 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11647,9 +11647,8 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11667,7 +11666,7 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) return false; continue; } @@ -11686,6 +11685,13 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return true; } +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_member_walk(env, btf, t, 0); +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -12066,7 +12072,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12082,7 +12088,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13138,7 +13144,7 @@ check_ok: break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13680,7 +13686,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14059,7 +14065,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), From 3fc3e7331af6f60979fdf55e776bfe67105b584a Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:29 -0700 Subject: [PATCH 045/660] bpf: Add btf_type_is_arena_ptr() Add btf_type_is_arena_ptr() to test whether a BTF type is a pointer carrying the "arena" type tag, and use it to simplify both btf_scan_type_tags() and btf_validate_return_type(). The helper has external linkage because a later patch calls it from verifier.c to decide whether a struct member returned by value may be an arena pointer. btf_scan_type_tags() drove btf_type_tag_walk() with a one-entry match table, so the "multiple type tags" error it checked for could never fire: btf_type_tag_walk() only fails when the walk sets more than one flag. Dropping that error path, and with it the int return, is therefore no functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061529.1693266-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 1 + kernel/bpf/btf.c | 62 ++++++++++++++++----------------------------- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..ddd0f4f32d24 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 47d43eb983a5..280530d25886 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -8106,9 +8090,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) From 8d7ea1b752e5e49b00442f27f2a76228423aa74b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:34 -0700 Subject: [PATCH 046/660] bpf: Let the by-value struct walk take the kinds of member it accepts btf_struct_member_walk() hardcodes that every member of a struct returned by value must be a scalar. A later patch needs the same walk to also accept arena pointers, so give it a mask of the member kinds it allows. The mask reaches callers through btf_struct_is_composed_of(), which replaces btf_type_is_scalar_struct() as the entry point exported to btf.c. btf_type_is_scalar_struct() stays as a verifier-local wrapper for the call sites that only ever ask about scalars. There is no functional change: btf_member_kind_allowed() is exactly btf_type_is_scalar() when the mask is BTF_MEMBER_SCALAR alone, and the shared check is the last statement of the loop body, so dropping the array branch's continue neither skips a member nor checks one twice. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061534.1693789-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 11 +++++++++-- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 38 ++++++++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3eb61edc8c5e..ae9f606539f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 280530d25886..b1f4ef614d4c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7979,7 +7979,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t)) + if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed077a5422af..90139f1b78d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,9 +11646,23 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec) + const struct btf_type *t, u32 member_kinds, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11666,7 +11680,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) return false; continue; } @@ -11675,21 +11689,25 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt if (!array->nelems) return false; member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; } - if (!btf_type_is_scalar(member_type)) + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } return true; } -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t) + const struct btf_type *t, u32 member_kinds) { - return btf_struct_member_walk(env, btf, t, 0); + return btf_struct_member_walk(env, btf, t, member_kinds, 0); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } enum kfunc_ptr_arg_type { From 9e0195cd1d312ed99c3107d82f418e97baa45289 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:39 -0700 Subject: [PATCH 047/660] bpf: Let a by-value struct nest arrays and structs freely Go through nested array types until it reaches a non-array type. Later code will check such non-array type based on current logic. A zero-length array is still rejected, at whichever dimension it appears. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061539.1695015-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 90139f1b78d1..a377b1995b51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11675,6 +11675,17 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt const struct btf_array *array; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) + return false; + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + } if (btf_type_is_struct(member_type)) { if (rec >= 3) { verbose(env, "max struct nesting depth exceeded\n"); @@ -11684,12 +11695,6 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt return false; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } From 282d07dc6c74a5b4eea19743dd8c487c4b8814da Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:44 -0700 Subject: [PATCH 048/660] bpf: Report which member makes a kfunc return type unsupported A kfunc that returns a struct by value may only return scalars, and the message that rejects one names the type but not the member at fault: kernel function bpf_kfunc_call_test_ret_ptr returns STRUCT prog_test_ret_ptr that is not composed of scalars Record the member that made the walk fail and name it. What is recorded is a path, not a single member, because the walk descends up to 4 levels. For struct outer { struct inner { void *p; } in; __u64 tag; }; naming 'p' alone would send the reader looking for a member struct outer does not have, so the actual message reads "member 'in.p' has type PTR". For struct s { void *p[2]; }; naming 'p' alone would blame the member, when it is the element type that was rejected, so the actual message reads "member 'p[]' has type PTR". A zero-length array has no element type to name and is rejected for its length instead, so it reports "member 'p' is a zero-length array". The detailed diagnostics for this failure: Verification failed: Program Structure: Unsupported kernel function return type Reason: bpf_kfunc_call_test_ret_ptr() returns STRUCT prog_test_ret_ptr by value. Its member 'p' is PTR, not a scalar. Only kfuncs returning scalar values, or structures composed of scalar values are supported. ... Suggestion: Call a kernel function that returns only scalars by value. A type nested deeper than the walk descends has no single member to blame, so that case reports the depth instead: Reason: bpf_kfunc_call_test_ret_deep() returns STRUCT prog_test_ret_deep by value. It nests structs more than 4 levels deep. ... Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061545.1695736-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 103 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a377b1995b51..745daf58b052 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,6 +11646,17 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, u32 member_kinds) { @@ -11662,10 +11673,12 @@ static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type * element type. */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, u32 member_kinds, int rec) + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11674,6 +11687,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); /* * Every element of an array is laid out in the value being @@ -11682,30 +11696,48 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt */ while (btf_type_is_array(member_type)) { array = btf_array(member_type); - if (!array->nelems) - return false; + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) - return false; + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, u32 member_kinds) { - return btf_struct_member_walk(env, btf, t, member_kinds, 0); + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); } static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, @@ -11715,6 +11747,25 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -14082,17 +14133,51 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + /* * The returned struct comes back as raw register bits modeled * as an unknown scalar, so it must contain only scalars: * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t)) { + if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, " Its member '%s' is %s, not a scalar.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values, or " + "structures composed of scalar values are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); return -EINVAL; } ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); From ff08e6b79ee1470e736054e9558c47e4150406b1 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:50 -0700 Subject: [PATCH 049/660] bpf: Allow a global function to return arena pointers by value A global function may already return an arena pointer on its own, and check_global_ret_scalar_reg() accepts one in either half of the R0:R2 pair. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Only a subprogram gets this: the main program returns to the kernel, which has no arena to cast the address back into, so btf_validate_return_type() keeps rejecting an arena pointer there, whether bare or inside a struct. The rejection message therefore has to name what the function at hand may return, or it would offer the main program an arena pointer it cannot have. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061550.1696869-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 28 +++++++++++++++---- .../selftests/bpf/progs/exceptions_fail.c | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index b1f4ef614d4c..9c2cab08bb79 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7973,13 +7973,23 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_struct(t) && t->size <= 16) { /* * A global function's caller models the return as an opaque - * scalar pair, so it may only return scalars by value. A local - * function is verified inline, so a pointer field stays tracked - * and needs no such restriction. + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. */ bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; - if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) return 0; } @@ -8073,10 +8083,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, "Global function %s() has unsupported return type. " - "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", - tname); + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9708efb93683..22503cf62e9f 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); From 9046109974dd1f3ffc3f62e22555be13eaf7fdb0 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:55 -0700 Subject: [PATCH 050/660] bpf: Allow arena pointers in a by-value kfunc return A kfunc may already return an arena pointer on its own, which the program casts back into the arena address space to use. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Such a member carries the arena type tag but not the address space qualifier, so the program has to cast it itself. Handing it back as a scalar costs nothing: an arena address has no provenance to track, and the cast is confined to the caller's arena. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061555.1697503-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 22 +++++++++++-------- .../selftests/bpf/progs/aggregate_ret_kfunc.c | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 745daf58b052..a725e3917976 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14138,13 +14138,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* * The returned struct comes back as raw register bits modeled - * as an unknown scalar, so it must contain only scalars: - * otherwise a pointer field would be laundered into a scalar - * and escape provenance and reference tracking. + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. */ - if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { verbose(env, - "kernel function %s returns %s %s that is not composed of scalars\n", + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); if (path.too_deep) { @@ -14165,16 +14168,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "member '%s' has type %s\n", bad_name, btf_type_str(path.bad_type)); member_note = bpf_diag_fmt( - env, " Its member '%s' is %s, not a scalar.", + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", bad_name, btf_type_str(path.bad_type)); } } bpf_diag_program_structure( env, insn_idx, "unsupported kernel function return type", - "Call a kernel function that returns only scalars by value.", + "Call a kernel function that returns only scalars or arena pointers by value.", "%s() returns %s %s by value.%s " - "Only kfuncs returning scalar values, or " - "structures composed of scalar values are " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " "supported.", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off), member_note); diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index d6b422ae9784..f10e5cf6fd89 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -71,7 +71,7 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 -__failure __msg("is not composed of scalars") +__failure __msg("is not composed of scalars or arena pointers") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( From cc03706d92c9948bc99702599872719e3dc29add Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:00 -0700 Subject: [PATCH 051/660] selftests/bpf: Check the member named for an unsupported kfunc return type Add tests to cover cases where a kfunc return type is rejected with proper messages including member names and array types. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061600.1697800-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/aggregate_ret_kfunc.c | 82 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 40 +++++++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 42 ++++++++++ 3 files changed, 164 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index f10e5cf6fd89..0361a7b42c9b 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -19,6 +19,11 @@ void __kfunc_btf_root(void) "r"(&bpf_kfunc_call_test_ret_fastcall), "r"(&bpf_kfunc_call_test_ret_ptr), "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_ptr_arr), + "r"(&bpf_kfunc_call_test_ret_deep), + "r"(&bpf_kfunc_call_test_ret_arr_struct), + "r"(&bpf_kfunc_call_test_ret_arr2d), "r"(&bpf_kfunc_call_test_ret_big)); } @@ -72,6 +77,7 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( @@ -84,6 +90,82 @@ __naked int aggregate_ret_kfunc_ptr_fail(void) : __clobber_all); } +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p[]' has type PTR") +__naked int aggregate_ret_kfunc_ptr_arr_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_ptr_arr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr_arr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in1[].in2.p' has type PTR") +__naked int aggregate_ret_kfunc_arr_struct_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr_struct];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr_struct) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'a[].p' has type PTR") +__naked int aggregate_ret_kfunc_arr2d_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr2d];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr2d) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("R2 !read_ok") diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 850cf4f830c4..dff1ad6f491f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,41 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) +{ + struct prog_test_ret_ptr_arr r = { .p = { NULL, NULL } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) +{ + struct prog_test_ret_arr_struct r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) +{ + struct prog_test_ret_arr2d r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + __bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) { struct prog_test_ret_ii r = { .a = a, .b = b }; @@ -1539,6 +1574,11 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 65e693ada736..9202c31ba6c9 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -70,6 +70,43 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_ptr_arr { /* 16 bytes: an array of pointers */ + void *p[2]; +}; + +struct prog_test_ret_arr_struct { /* 16 bytes: the pointer is under an array of structs */ + struct { + struct { + void *p; + } in2; + } in1[2]; +}; + +struct prog_test_ret_arr2d { /* 16 bytes: a two dimensional array of structs */ + struct { + void *p; + } a[1][2]; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ __u64 a; __u64 b; @@ -159,6 +196,11 @@ struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; +struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; +struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, From 18bc6183884279b279026737b1fa7cc0517010bc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:05 -0700 Subject: [PATCH 052/660] selftests/bpf: Test global functions returning arena pointers by value Cover the by-value struct returns a global function may now make: two arena pointers filling R0:R2, an arena pointer beside a scalar, an array of them, and an eight byte struct returned in R0 alone. The existing cases for a struct and a union carrying a plain pointer stay rejected. check_arena_struct_ret() also stores through both halves of the returned pair and reads them back, so the test covers the returned pointers still being usable as arena pointers rather than only the program verifying. A toolchain guard goes into each file, and they differ because the constructs do. aggregate_ret_func.c uses "#if defined(__clang__)". Those cases are __naked, so the function body is the inline asm and nothing else: the clang compiler never lowers the return, and the 16 byte return type reaches the verifier only through BTF. The minimum checked is LLVM 21. gcc is excluded because it returns a by-value struct through a hidden pointer and emits the 'r0 = r1' returning it after the __naked body's exit, leaving the subprogram falling through. verifier_arena.c uses "#if defined(__clang_major__) && __clang_major__ >= 23". arena_word_pair() returns the struct from C, so the compiler lowers the return itself, and a 16 byte return only lands in R0:R2 with the LLVM 23 BPF ABI. An older clang does not fall back to anything here, it rejects the function with "aggregate returns are not supported", so the floor is what lets the file build at all. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061605.1698601-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/aggregate_ret_func.c | 146 ++++++++++++++++++ .../selftests/bpf/progs/verifier_arena.c | 48 ++++++ 2 files changed, 194 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c index 6f66fc822ced..f0077d951a25 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include +#include #include "bpf_misc.h" typedef unsigned __int128 u128; @@ -234,4 +235,149 @@ __naked int aggregate_ret_global_union_ptr_fail(void) #endif +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +union arena_upair { + void __arena *p; + __u64 halves[2]; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +__naked union arena_upair global_ret_arena_union(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_union(void) +{ + asm volatile ( + "call %[global_ret_arena_union];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_union) + : __clobber_all); +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b0..332322c9b54c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -734,4 +734,52 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_word_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_word_pair arena_word_pair(u32 __arena *page) +{ + struct arena_word_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *first, *second; + struct arena_word_pair pair; + + if (!page) + return 1; + + pair = arena_word_pair(page); + if (!pair.first || !pair.second) + return 2; + + /* Both halves of the pair must still be usable as arena pointers. */ + first = pair.first; + second = pair.second; + *first = 1; + *second = 2; + if (*first != 1) + return 3; + if (*second != 2) + return 4; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; From f7213961e8e24b3ed98b13406c06fcf91138ad2f Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:10 -0700 Subject: [PATCH 053/660] selftests/bpf: Test kfuncs returning arena pointers by value Cover the by-value struct returns a kfunc may now make: two arena pointers filling R0:R2, and an arena pointer beside a scalar. Two further cases drop the tag from one member of a struct and one arm of a union, and stay rejected naming that member, so what decides is the tag rather than the member being a pointer. The existing cases for a struct and a nested struct carrying a plain pointer stay rejected as well. These cases call the kfuncs from C, so the compiler lowers the by-value return itself, and a struct or union only lands in R0:R2 with the LLVM 23 BPF ABI. An older clang, and gcc, return it through a hidden pointer in R1 instead, which shifts the arguments along and fails verification. The file is therefore guarded on LLVM 23, falling back to a dummy test. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061610.1699950-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_ret.c | 42 ++++++ .../bpf/progs/aggregate_ret_kfunc_arena.c | 121 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 32 +++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 30 +++++ 4 files changed, 225 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c index e0b94ed10f94..07d9d6e1d6b8 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -1,11 +1,53 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include #include "aggregate_ret_func.skel.h" #include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} void test_aggregate_ret(void) { RUN_TESTS(aggregate_ret_func); RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 000000000000..a0f263694a63 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena(void *ctx) +{ + u32 volatile __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *a, *b; + struct prog_test_ret_arena r; + + if (!page) + return 1; + + /* Both halves come back in R0:R2, pointing at page and page + 4. */ + r = bpf_kfunc_call_test_ret_arena((u64)page); + if (!r.a || !r.b) + return 2; + + a = (u32 __arena *)r.a; + b = (u32 __arena *)r.b; + *a = 1; + *b = 2; + if (*a != 1) + return 3; + if (*b != 2) + return 4; + + page[0] = 7; + if (*a != 7) + return 5; + page[1] = 9; + if (*b != 9) + return 6; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena_mixed(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct prog_test_ret_arena_mixed r; + u32 volatile __arena *p; + + if (!page) + return 1; + + /* An arena pointer in R0 beside a scalar in R2. */ + r = bpf_kfunc_call_test_ret_arena_mixed((u64)page); + if (!r.p) + return 2; + if (r.tag != 0xbeef) + return 3; + + p = (u32 __arena *)r.p; + *p = 3; + if (*p != 3) + return 4; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_untagged_fail(void *ctx) +{ + struct prog_test_ret_arena_untagged r; + + r = bpf_kfunc_call_test_ret_arena_untagged(0); + + return r.a == r.b; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_union_fail(void *ctx) +{ + union prog_test_ret_arena_union r; + + r = bpf_kfunc_call_test_ret_arena_union(0); + + return r.a == r.b; +} + +#else + +SEC("socket") +__description("aggregate_ret_kfunc_arena: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index dff1ad6f491f..2380b6cbdead 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,34 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(u64 addr) +{ + struct prog_test_ret_arena r = { .a = (void *)addr, .b = (void *)(addr + 4) }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 addr) +{ + struct prog_test_ret_arena_mixed r = { .p = (void *)addr, .tag = 0xbeef }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(u64 addr) +{ + struct prog_test_ret_arena_untagged r = { .a = (void *)addr, .b = NULL }; + + return r; +} + +__bpf_kfunc union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(u64 addr) +{ + union prog_test_ret_arena_union r = { .a = (void *)addr }; + + return r; +} + __bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) { struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; @@ -1574,6 +1602,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_untagged) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_union) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 9202c31ba6c9..b213ef14848b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -70,6 +76,26 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_arena_untagged { /* 16 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +union prog_test_ret_arena_union { /* 8 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ struct { void *p; @@ -200,6 +226,10 @@ struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(__u64 addr) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 addr) __ksym; +struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(__u64 addr) __ksym; +union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(__u64 addr) __ksym; struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, From 25b0d73a7742d7415156c607f6278682efc12fc2 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:15 -0700 Subject: [PATCH 054/660] docs/bpf: Document arena pointers in a by-value return Section 2.9 describes the by-value return contract as scalars only, which no longer holds: a kfunc and a global subprogram may now return a struct or union whose members are scalars or arena pointers. Update it. Note that an arena pointer member is handed back as a scalar like every other member. That costs nothing: the program has to cast_kern() the value before it can be used, and the verifier allows that cast on any scalar, so the member gives the program no reach it did not already have. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061615.1700421-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- Documentation/bpf/kfuncs.rst | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 89dea6b0b024..ebe37c1fe9d2 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -581,20 +581,29 @@ against the arena. Larger accesses must verify the range explicitly. A kfunc may return a scalar, a pointer, or a small struct or union by value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. -A struct or union returned by value must be composed only of scalars -(recursively), where a scalar is an integer or an enum; arrays of scalars are -allowed as members. Its bytes are handed back to the program as the raw -contents of R0 (and R2), so a pointer field would be laundered into a scalar -and escape the verifier's pointer provenance and reference tracking. A struct -or union with a pointer member is therefore rejected at load time, and so is -one with a floating-point member, which the ABI may not return in R0:R2 at +A struct or union returned by value must be composed only of scalars and arena +pointers, where a scalar is an integer or an enum and an arena pointer is one +carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in +structs and unions and in arrays of any number of dimensions, in any +combination, as long as what the nesting bottoms out in is a scalar or an arena +pointer. Its bytes are handed back to the program as the raw contents of R0 +(and R2), so a member of any other pointer type would be laundered into a +scalar and escape the verifier's pointer provenance and reference tracking. A +struct or union with such a member is therefore rejected at load time, and so +is one with a floating-point member, which the ABI may not return in R0:R2 at all. +An arena pointer member is handed back as a scalar too, but nothing is lost by +that. The program must ``cast_kern()`` the value before it can be used, and the +verifier allows that cast on any scalar, so a laundered arena address gives the +program no reach it did not already have. The result is confined to the +program's arena in either case. + A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a -scalar-only struct or union, or an ``__int128``. Such a value is returned -in the register pair R0:R2, matching the convention LLVM uses for the BPF -target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or -union of 8 bytes or less is returned in R0 alone. +struct or union of scalars and arena pointers, or an ``__int128``. Such a +value is returned in the register pair R0:R2, matching the convention LLVM +uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2. +A struct or union of 8 bytes or less is returned in R0 alone. :: @@ -620,10 +629,10 @@ used when the program is JITed, since the interpreter propagates only R0 out of a subprogram: without a JIT the return value stays in R0 alone, and a caller reading R2 is rejected for reading an uninitialized register. A global subprogram is verified in isolation, so its by-value struct or union return is -restricted to scalars just like a kfunc's; a static subprogram is verified -inline and has no such restriction. The main program is not covered: its return -value is the program's exit code, read out of R0 alone, so a declared upper -half is never looked at. +restricted to scalars and arena pointers just like a kfunc's; a static +subprogram is verified inline and has no such restriction. The main program is +not covered: its return value is the program's exit code, read out of R0 +alone, so a declared upper half is never looked at. A global subprogram must leave a scalar in *every* register of the pair, so both halves of the returned value have to be assigned. Leaving the upper half From 264d8fd2794fabe76a89abd5d4f5f8c1ed5fad85 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:17 +0200 Subject: [PATCH 055/660] bpf, keys: Add a bpf keyring for program signature validation BPF program signatures can currently be verified against one of the system keyrings (builtin, secondary, platform) or against an arbitrary user/session caller-supplied keyring named through keyring_id. There is nothing in between: the system keyrings need a kernel rebuild or a vouched-for enrollment to rotate a key, while a caller-supplied keyring is fully controlled by the loader and therefore carries no trust on its own (unless explicitly combined with BPF LSM to protect against key tampering). Add a dedicated bpf keyring to fill that gap, modelled after the dm-verity keyring which was added in commit 033724b1c627 ("dm-verity: add dm-verity keyring") and which can eventually be used also via systemd [0] through the same enrollment method as in dm-verity's case. It is selected with the new KEY_SPEC_BPF_KEYRING special key id and gives an operator a place to enroll a BPF-only signing key at boot, specifically scoped to BPF program loading and nothing else in the kernel's trust hierarchy. The id is reserved from the KEY_SPEC space so that the latter is not linked into any process keyring, and lookup_user_key() resolves KEY_SPEC_BPF_KEYRING constant instead of having to look it up via /proc/keys first. By default the keyring is sealed empty at init. Systems that want to provision keys pass bpf.keyring_unsealed=1, which leaves the keyring open for the initrd to add keys to. The keyring is only ever consulted once it is both non-empty and restricted. An unrestricted keyring is ignored. Signed-off-by: Daniel Borkmann Link: https://github.com/systemd/systemd/pull/43549 [0] Link: https://lore.kernel.org/r/20260828175227.1537793-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../admin-guide/kernel-parameters.txt | 8 +++ include/linux/bpf.h | 7 ++ include/linux/key.h | 2 + include/uapi/linux/keyctl.h | 1 + kernel/bpf/Makefile | 3 + kernel/bpf/keys.c | 68 +++++++++++++++++++ kernel/bpf/verifier.c | 9 +++ security/keys/process_keys.c | 25 +++++++ 8 files changed, 123 insertions(+) create mode 100644 kernel/bpf/keys.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e4643634a9b1..2beb61092bb3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -667,6 +667,14 @@ Kernel parameters See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c2027688be3e..039674b2f875 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1735,6 +1735,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -3819,6 +3820,7 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3839,6 +3841,11 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..bd10fe45819d 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -440,6 +440,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..9a92c348bbda 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..08549b3220c1 --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a725e3917976..0c229a226eab 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -21127,6 +21128,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21164,9 +21167,15 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return -EINVAL; if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); + else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + key = bpf_lookup_keyring(); else key = bpf_lookup_user_key(attr->keyring_id, 0); if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index a63c46bb2d14..44358388e395 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -590,6 +593,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -741,6 +758,14 @@ try_again: key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1) From abaa0835b4f381067cc04bfd58180679d813578b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:18 +0200 Subject: [PATCH 056/660] bpf: Refuse caller-supplied keyrings when the bpf one is active Nothing changes for systems that do not use the bpf keyring. Without bpf.keyring_unsealed=1 a caller-supplied keyring behaves exactly as before, which also lets it serve as the staging step for software installed onto a running system whose signing key is not enrolled anywhere yet. Passing bpf.keyring_unsealed=1 states that the bpf keyring is the trust anchor for this boot, so from the first program load onwards a caller- supplied keyring is refused with -EPERM. Deriving this from the boot flag rather than from the keyring's runtime state keeps the decision immutable from userspace. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++++++ include/linux/bpf.h | 6 ++++++ kernel/bpf/keys.c | 5 +++++ kernel/bpf/verifier.c | 10 +++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2beb61092bb3..e70fb15757d4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -673,6 +673,14 @@ Kernel parameters keys. Once the keyring is restricted it becomes active and can be used for BPF program signature verification. + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + See Documentation/bpf/signing.rst bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 039674b2f875..3a7eb2185c35 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3821,6 +3821,7 @@ struct bpf_key { struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3846,6 +3847,11 @@ static inline struct bpf_key *bpf_lookup_keyring(void) return NULL; } +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c index 08549b3220c1..bffc356839ac 100644 --- a/kernel/bpf/keys.c +++ b/kernel/bpf/keys.c @@ -18,6 +18,11 @@ static bool bpf_keyring_unsealed __ro_after_init; module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + struct bpf_key *bpf_lookup_keyring(void) { struct bpf_key *bkey; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c229a226eab..dab618836299 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21165,12 +21165,16 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, if (!attr->signature_size || attr->signature_size > KMALLOC_MAX_CACHE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { key = bpf_lookup_keyring(); - else + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { verbose(env, "the bpf keyring is empty or has not been restricted\n"); From 300b348e6d5e8e96c065c42c9999baca224c1661 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:19 +0200 Subject: [PATCH 057/660] bpf: Raise the bound on a program's signature size signature_size is bounded by KMALLOC_MAX_CACHE_SIZE, which is 8 KiB on a 4 KiB page system. Back then we chose it somewhat arbitrarily and was picked when a BPF program signature was RSA or ECDSA. ML-DSA (FIPS-204) verification is wired through the X.509 and PKCS#7 parsers, and BPF reaches them too via verify_pkcs7_signature() without having to know the concrete algorithm. The bound becomes a bit too small, thus add an explicit BPF_PROG_MAX_SIGNATURE_SIZE of 64 KiB and use that instead to cover all options. KMALLOC_MAX_CACHE_SIZE is PAGE_SIZE-derived, so what was accepted so far depended on the page size which is not optimal as it should be the same behavior on every configuration. On 64 KiB page kernels this lowers the ceiling from 128 KiB to 64 KiB. Nothing that could have been verified is affected as the largest signature the kernel implements is ML-DSA-87 at 4627 bytes. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-4-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 15 ++++++++++----- .../selftests/bpf/prog_tests/signed_loader.c | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dab618836299..8f585ceb2cd5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21119,6 +21119,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -21158,13 +21166,10 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, u64 data_sz; int err = 0; - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 77381d345435..0c5294738d6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -571,8 +571,9 @@ static void signature_too_large(void) if (gen_loader_fixture_init(&f) == 0) { /* - * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) - * is rejected before the buffer is read. + * signature_size beyond the kernel's bound + * (BPF_PROG_MAX_SIGNATURE_SIZE) is rejected before the buffer + * is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); From d37168c99361b6ee36d5be0c1e159b594f32473c Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:20 +0200 Subject: [PATCH 058/660] bpftool: Support ML-DSA program signing Add bpftool support for ML-DSA program signing and drop the flag for ML-DSA keys on affected OpenSSL versions, the same way as commit 0ad9a71933e7 ("modsign: Enable ML-DSA module signing"). Also, an ML-DSA-87 signature is 4627 bytes on its own, so the blob does not fit into the 4 KiB of MAX_SIG_SIZE anymore and signing would fail otherwise. Bump to 16 KiB. MAX_SIG_SIZE only sizes the buffer for what bpftool itself emits (unrelated to BPF_PROG_MAX_SIGNATURE_SIZE). Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-5-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/main.h | 2 +- tools/bpf/bpftool/sign.c | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 78b6e0ebb85d..9315a1db1f7c 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -57,7 +57,7 @@ static inline void *u64_to_ptr(__u64 ptr) }) #define ERR_MAX_LEN 1024 -#define MAX_SIG_SIZE 4096 +#define MAX_SIG_SIZE 16384 #define BPF_TAG_FMT "%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx" diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index 88726a6db6d0..adbfd4ae5c02 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -130,6 +130,12 @@ cleanup: int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) { + unsigned int signer_flags = CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | +#ifdef CMS_NO_SIGNING_TIME + CMS_NO_SIGNING_TIME | +#endif + CMS_USE_KEYID | CMS_NOATTR; + const EVP_MD *cms_digest = EVP_sha256(); BIO *bd_in = NULL, *bd_out = NULL; EVP_PKEY *private_key = NULL; CMS_ContentInfo *cms = NULL; @@ -167,6 +173,20 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L + if (EVP_PKEY_is_a(private_key, "ML-DSA-44") || + EVP_PKEY_is_a(private_key, "ML-DSA-65") || + EVP_PKEY_is_a(private_key, "ML-DSA-87")) { + /* + * See workaround in 0ad9a71933e7 ("modsign: Enable ML-DSA + * module signing"). Kernel only accepts sha512, see also + * 8bbdeb7a25b4 ("pkcs7, x509: Add ML-DSA support"). + */ + signer_flags &= ~CMS_NOATTR; + cms_digest = EVP_sha512(); + } +#endif + cms = CMS_sign(NULL, NULL, NULL, NULL, CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_DETACHED | CMS_STREAM); @@ -175,9 +195,7 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - if (!CMS_add1_signer(cms, x509, private_key, EVP_sha256(), - CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | - CMS_USE_KEYID | CMS_NOATTR)) { + if (!CMS_add1_signer(cms, x509, private_key, cms_digest, signer_flags)) { err = -EINVAL; goto cleanup; } From 6754aeca602224a3f5e8c6703c6549791d8f6227 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:21 +0200 Subject: [PATCH 059/660] selftests/bpf: Add a test for the sealed bpf keyring bpf_keyring_sealed checks that a load naming the bpf keyring fails while the keyring has not been provisioned. It uses a junk signature as the size check and the keyring lookup both happen before any crypto, so the error under test is reached without a real signature and the ordering is what gets verified: # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/9 signed_loader/signed_module_kfunc_rejected:OK #425/10 signed_loader/signature_failure_logs:OK #425/11 signed_loader/signature_too_large:OK #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK #425/15 signed_loader/metadata_ctx_max_entries_ignored:OK #425/16 signed_loader/metadata_ctx_initial_value_ignored:OK #425/17 signed_loader/signature_authenticates_insns:OK #425/18 signed_loader/signature_authenticates_metadata:OK #425/19 signed_loader/hash_requires_frozen:OK [...] #425 signed_loader:OK Summary: 1/30 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-6-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 0c5294738d6c..e30166c00be6 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -32,8 +32,13 @@ enum { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; +#ifndef KEY_SPEC_BPF_KEYRING +#define KEY_SPEC_BPF_KEYRING -9 +#endif + static int load_loader(const void *insns, __u32 insns_sz, int map_fd, const void *sig, __u32 sig_sz, __s32 keyring_id, __u32 fd_array_cnt) @@ -627,6 +632,83 @@ static void signature_bad_keyring(void) gen_loader_fixture_fini(&f); } +static bool keyring_unsealed_boot(void) +{ + char val = 0; + int fd; + + fd = open("/sys/module/bpf/parameters/keyring_unsealed", O_RDONLY); + if (fd < 0) + return false; + if (read(fd, &val, 1) != 1) + val = 0; + close(fd); + return val == 'Y' || val == '1'; +} + +static int bpf_keyring_lookup(int *nr_keys) +{ + char line[512], type[32], desc[64]; + int serial = -ENOENT; + FILE *f; + + f = fopen("/proc/keys", "r"); + if (!f) + return -errno; + + while (fgets(line, sizeof(line), f)) { + unsigned int hex; + char *sum; + + if (sscanf(line, "%x %*s %*s %*s %*s %*s %*s %31s %63s", + &hex, type, desc) != 3) + continue; + if (strcmp(type, "keyring") || strcmp(desc, ".bpf:")) + continue; + + serial = (int)hex; + if (nr_keys) { + sum = strstr(line, ".bpf: "); + *nr_keys = !sum || !strncmp(sum + 6, "empty", 5) ? + 0 : atoi(sum + 6); + } + break; + } + fclose(f); + return serial; +} + +static void bpf_keyring_sealed(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int serial, key, fd; + + if (keyring_unsealed_boot()) { + printf("%s:SKIP:the bpf keyring was unsealed at boot\n", __func__); + test__skip(); + return; + } + serial = bpf_keyring_lookup(NULL); + if (serial >= 0) { + key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, + KEY_SPEC_BPF_KEYRING); + if (key >= 0) + syscall(__NR_keyctl, KEYCTL_UNLINK, key, + KEY_SPEC_BPF_KEYRING); + ASSERT_EQ(key < 0 ? -errno : 0, -EPERM, + "nothing links into a sealed keyring"); + } + if (gen_loader_fixture_init(&f) == 0) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_BPF_KEYRING, 0); + ASSERT_EQ(fd, -ENOKEY, "sealed bpf keyring rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -1806,6 +1888,8 @@ void test_signed_loader(void) signature_zero_size(); if (test__start_subtest("signature_bad_keyring")) signature_bad_keyring(); + if (test__start_subtest("bpf_keyring_sealed")) + bpf_keyring_sealed(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) metadata_ctx_max_entries_ignored(); if (test__start_subtest("metadata_ctx_initial_value_ignored")) From e7ada0800ce2a6dcc35ca0e1dfdab37bfb2ea4a9 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:22 +0200 Subject: [PATCH 060/660] selftests/bpf: Rebuild signed lskels when signing key changes The signing key is regenerated whenever verify_sig_setup.sh changes, but the signed light skeletons only depend on the BPF object and on bpftool, not on the key they are signed with. Thus, add the certificate as a prereq so a new key forces the skeletons to be signed again. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-7-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 5f1a3bfc0569..05b3ee64290f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -663,7 +663,7 @@ $(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) }) && \ rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) +$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) | $(TRUNNER_OUTPUT) $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ $$(RM) $$@; \ printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ From 1841d6cbc5c777d5dfba3723965ab14a9764d96e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:23 +0200 Subject: [PATCH 061/660] selftests/bpf: Rename the verify_sig_setup.sh setup into setup-rsa The script's "setup" action generates an RSA key, enrolls it and builds a keyring around it. The name says nothing about the algorithm, which is fine while there is only one, but we'll add "setup-mldsa" soon, therefore rename the existing one into "setup-rsa". No functional change. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-8-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 18 +++++++++--------- .../bpf/prog_tests/verify_pkcs7_sig.c | 4 ++-- .../testing/selftests/bpf/verify_sig_setup.sh | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index e30166c00be6..fbf14c7dd560 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -462,7 +462,7 @@ static void signed_btf_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -914,7 +914,7 @@ static void signature_authenticates_insns(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1014,7 +1014,7 @@ static void signature_authenticates_metadata(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1350,7 +1350,7 @@ static void lsm_signature_verdict(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); dir = NULL; goto out; @@ -1533,7 +1533,7 @@ static void loadtime_verify(struct bpf_object *obj, int expect_maps) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1631,7 +1631,7 @@ static void signed_no_fd_array(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1702,7 +1702,7 @@ static void signed_map_by_fd_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1764,7 +1764,7 @@ static void signed_sparse_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1818,7 +1818,7 @@ static void signed_module_kfunc_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index f327feb8e38c..12b146d205d7 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -257,7 +257,7 @@ static void test_verify_pkcs7_sig_from_map(void) if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) return; - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto close_prog; @@ -458,7 +458,7 @@ static void test_pkcs7_sig_fsverity(void) snprintf(data_path, PATH_MAX, "%s/data-file", tmp_dir); snprintf(sig_path, PATH_MAX, "%s/sig-file", tmp_dir); - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto out; diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 09179fb551f0..202e6e6418fe 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -28,7 +28,7 @@ authorityKeyIdentifier=keyid usage() { - echo "Usage: $0 " + echo "Usage: $0 " exit 1 } @@ -47,7 +47,7 @@ genkey() ${tmp_dir}/signing_key.der -outform der } -setup() +setup_rsa() { local tmp_dir="$1" @@ -108,8 +108,8 @@ main() [[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1 - if [[ "${action}" == "setup" ]]; then - setup "${tmp_dir}" + if [[ "${action}" == "setup-rsa" ]]; then + setup_rsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then From 478cf384def8cc4cb7f92739b73c76acbac976ca Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:24 +0200 Subject: [PATCH 062/660] selftests/bpf: Add an end-to-end ML-DSA signed loader test The BPF signing is algorithm agnostic, but so far the BPF CI only has tested a single one. BPF hands verify_pkcs7_signature() a keyring and byte ranges, and everything below it already understands ML-DSA, so add a test for ML-DSA signed program to validate it works as well. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/10 signed_loader/signature_failure_logs:OK #425/11 signed_loader/signature_too_large:OK #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK #425/15 signed_loader/mldsa_signed_load:OK #425/16 signed_loader/metadata_ctx_max_entries_ignored:OK #425/17 signed_loader/metadata_ctx_initial_value_ignored:OK #425/18 signed_loader/signature_authenticates_insns:OK #425/19 signed_loader/signature_authenticates_metadata:OK #425/20 signed_loader/hash_requires_frozen:OK [...] #425 signed_loader:OK Summary: 1/31 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-9-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/config | 2 + .../selftests/bpf/prog_tests/signed_loader.c | 125 +++++++++++++++++- .../testing/selftests/bpf/verify_sig_setup.sh | 57 +++++++- 3 files changed, 177 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index ea7044f30adc..2f79688dcf7c 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -12,7 +12,9 @@ CONFIG_BPF_SYSCALL=y # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set CONFIG_CGROUP_BPF=y CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MLDSA=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y CONFIG_CRYPTO_USER_API=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index fbf14c7dd560..87f155de0e65 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -39,6 +39,12 @@ enum { #define KEY_SPEC_BPF_KEYRING -9 #endif +/* verify_sig_setup.sh exits with this when openssl cannot do ML-DSA. */ +#define SETUP_SKIP (-77) + +/* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ +#define MLDSA87_SIGNATURE_SIZE 4627 + static int load_loader(const void *insns, __u32 insns_sz, int map_fd, const void *sig, __u32 sig_sz, __s32 keyring_id, __u32 fd_array_cnt) @@ -161,12 +167,30 @@ static int run_setup(const char *cmd, const char *dir) } if (waitpid(pid, &status, 0) < 0) return -errno; - return (WIFEXITED(status) && - WEXITSTATUS(status) == 0) ? 0 : -EINVAL; + if (!WIFEXITED(status)) + return -EINVAL; + return -WEXITSTATUS(status); } -static int sign_buf(const char *dir, const void *buf, __u32 len, - void *sig, __u32 *sig_sz) +static void genkey_dir_fini(const char *dir) +{ + static const char * const files[] = { + "signing_key.der", "signing_key.pem", "x509.genkey", + }; + char path[PATH_MAX]; + size_t i; + + if (!dir) + return; + for (i = 0; i < ARRAY_SIZE(files); i++) { + snprintf(path, sizeof(path), "%s/%s", dir, files[i]); + unlink(path); + } + rmdir(dir); +} + +static int sign_buf_digest(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz, const char *digest) { char data_tmpl[PATH_MAX], key[PATH_MAX]; char sigpath[PATH_MAX + sizeof(".p7s")]; @@ -195,7 +219,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, } if (pid == 0) { snprintf(key, sizeof(key), "%s/signing_key.pem", dir); - execlp("./sign-file", "./sign-file", "-d", "sha256", + execlp("./sign-file", "./sign-file", "-d", digest, key, key, data_tmpl, NULL); exit(1); } @@ -233,6 +257,12 @@ out: return ret; } +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + return sign_buf_digest(dir, buf, len, sig, sig_sz, "sha256"); +} + struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -1607,6 +1637,89 @@ static void loadtime_with_map(void) test_signed_loader_map__destroy(skel); } +/* + * End-to-end signed load with a post-quantum key. ML-DSA (FIPS-204) is wired + * through the X.509 and PKCS#7 parsers, and BPF reaches them via + * verify_pkcs7_signature() without knowing the algorithm, so an ML-DSA key in + * the keyring should verify an ML-DSA signed program with no BPF-side work. + */ +static void mldsa_signed_load(void) +{ + char dir_tmpl[] = "/tmp/bpfmldsaXXXXXX"; + int map_fd = -1, prog_fd = -1, err; + __u8 *sig = NULL, *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + __u32 sig_sz = 16384; + char *dir; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + + err = run_setup("setup-mldsa", dir); + if (err == SETUP_SKIP) { + printf("%s:SKIP:no working ML-DSA signing, set SELFTESTS_VERBOSE=1\n", + __func__); + test__skip(); + genkey_dir_fini(dir); + return; + } + if (!ASSERT_OK(err, "verify_sig_setup setup-mldsa")) { + genkey_dir_fini(dir); + return; + } + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + + /* + * ML-DSA hashes the message itself, but openssl before 4.0 cannot + * produce a CMS message without signedAttrs for it, and with those in + * play only SHA-512 is permitted for the messageDigest attribute. + */ + if (!ASSERT_OK(sign_buf_digest(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &sig_sz, "sha512"), + "sign insns||metadata with ML-DSA")) + goto out; + + /* + * Guard against the setup silently handing back some other key type: + * an RSA or ECDSA signature is a few hundred bytes, where an ML-DSA-87 + * one cannot be smaller than the raw signature it carries. + */ + ASSERT_GT(sig_sz, MLDSA87_SIGNATURE_SIZE, "ML-DSA-87 signature size"); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_OK_FD(prog_fd, "ML-DSA signed loader load"); +out: + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + free(buf); + free(sig); + run_setup("cleanup", dir); +} + /* * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL * program with no fd_array is signed over its instructions alone: the kernel @@ -1890,6 +2003,8 @@ void test_signed_loader(void) signature_bad_keyring(); if (test__start_subtest("bpf_keyring_sealed")) bpf_keyring_sealed(); + if (test__start_subtest("mldsa_signed_load")) + mldsa_signed_load(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) metadata_ctx_max_entries_ignored(); if (test__start_subtest("metadata_ctx_initial_value_ignored")) diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 202e6e6418fe..ba5921e4b12d 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -28,7 +28,7 @@ authorityKeyIdentifier=keyid usage() { - echo "Usage: $0 " + echo "Usage: $0 " exit 1 } @@ -57,6 +57,57 @@ setup_rsa() keyctl link $key_id $keyring_id } +mldsa_supported() +{ + local tmp_dir="$1" + + genkey_mldsa "${tmp_dir}" || return 1 + : > ${tmp_dir}/probe + # Same digest as the caller signs with, see sign_buf_digest(). + ./sign-file -d sha512 ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.pem ${tmp_dir}/probe || return 1 + rm -f ${tmp_dir}/probe ${tmp_dir}/probe.p7s +} + +genkey_mldsa() +{ + local tmp_dir="$1" + + echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey + + # No - here: ML-DSA hashes the message itself, so openssl + # ignores an explicit digest for it. + openssl req -new -nodes -utf8 -days 36500 \ + -batch -x509 -newkey ML-DSA-87 \ + -config ${tmp_dir}/x509.genkey \ + -outform PEM -out ${tmp_dir}/signing_key.pem \ + -keyout ${tmp_dir}/signing_key.pem 2>&1 + + openssl x509 -in ${tmp_dir}/signing_key.pem -out \ + ${tmp_dir}/signing_key.der -outform der +} + +mldsa_skip() +{ + local tmp_dir="$1" + + rm -f ${tmp_dir}/x509.genkey ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.der ${tmp_dir}/probe \ + ${tmp_dir}/probe.p7s + exit 77 +} + +setup_mldsa() +{ + local tmp_dir="$1" + + mldsa_supported "${tmp_dir}" || mldsa_skip "${tmp_dir}" + key_id=$(cat ${tmp_dir}/signing_key.der | + keyctl padd asymmetric ebpf_testing_key @s) + keyring_id=$(keyctl newring ebpf_testing_keyring @s) + keyctl link $key_id $keyring_id +} + cleanup() { local tmp_dir="$1" @@ -91,7 +142,7 @@ catch() local exit_code="$1" local log_file="$2" - if [[ "${exit_code}" -ne 0 ]]; then + if [[ "${exit_code}" -ne 0 && "${exit_code}" -ne 77 ]]; then cat "${log_file}" >&3 fi @@ -110,6 +161,8 @@ main() if [[ "${action}" == "setup-rsa" ]]; then setup_rsa "${tmp_dir}" + elif [[ "${action}" == "setup-mldsa" ]]; then + setup_mldsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then From b638a82d64f1ee480c4b6c6c8e3da2ae84c1edb0 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:25 +0200 Subject: [PATCH 063/660] selftests/bpf: Allow appending to guest kernel cmdline in vmtest.sh vmtest.sh currently hardcodes the guest command line, so there is no way to ask for such a setting without editing the script. Append $KERNEL_CMDLINE_EXTRA when set so it can be used for testing the BPF keyring: # KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" \ ./vmtest.sh -- ./test_progs -t signed_loader Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-10-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/vmtest.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 6a3d026d76bd..e7e0b419a0b8 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -107,6 +107,14 @@ Options: -s) Instead of powering off the VM, start an interactive shell. If is specified, the shell runs after the command finishes executing + +Environment variables: + + KERNEL_CMDLINE_EXTRA + Extra arguments to append to the guest kernel command + line, for tests that need a boot-time setting. e.g: + + KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" $0 -- ./test_progs -t signed_loader EOF } @@ -286,6 +294,12 @@ EOF QEMU_FLAGS=("${HOST_FLAGS[@]}") fi + local kernel_cmdline="root=/dev/vda rw console=${QEMU_CONSOLE}" + + if [[ -n "${KERNEL_CMDLINE_EXTRA:-}" ]]; then + kernel_cmdline+=" ${KERNEL_CMDLINE_EXTRA}" + fi + ${QEMU_BINARY} \ -nodefaults \ -display none \ @@ -294,7 +308,7 @@ EOF -m 4G \ -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \ -kernel "${kernel_bzimage}" \ - -append "root=/dev/vda rw console=${QEMU_CONSOLE}" + -append "${kernel_cmdline}" } copy_logs() From 1c8cb8bf73293264e2a44d0b029a5047a7dd5e20 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:26 +0200 Subject: [PATCH 064/660] selftests/bpf: Add tests for bpf keyring in signed loader bpf_keyring_provisioned walks the keyring through its whole lifecycle in one boot for ease of testing. It enrolls a freshly generated key into the bpf keyring, confirms a load is still refused with -ENOKEY while the keyring carries no restriction, then restricts it, and only then does the same signed BPF program load with the bpf keyring. A caller-supplied keyring is asserted to be refused both before and after the restriction, since what refuses it is bpf.keyring_unsealed=1 rather than the state of the keyring. Unsealing is a boot-time decision which also refuses the caller-supplied keyrings that most subtests here sign against, so each subtest is tagged with the boot it needs and the ones which cannot run report as skipped instead of being dropped. Regular run: # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK [...] #425/30 signed_loader/signed_map_by_fd_rejected:OK #425/31 signed_loader/signed_sparse_fd_array_rejected:OK #425/32 signed_loader/bpf_keyring_provisioned:SKIP #425 signed_loader:OK (SKIP: 1/32) Summary: 1/31 PASSED, 1 SKIPPED, 0/0 FAILED Unsealed run: # KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" \ LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader #425/1 signed_loader/loadtime_no_map:SKIP #425/2 signed_loader/loadtime_with_map:SKIP #425/3 signed_loader/metadata_match:OK [...] #425/30 signed_loader/signed_map_by_fd_rejected:SKIP #425/31 signed_loader/signed_sparse_fd_array_rejected:SKIP #425/32 signed_loader/bpf_keyring_provisioned:OK #425 signed_loader:OK (SKIP: 17/32) Summary: 1/15 PASSED, 17 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-11-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 376 +++++++++++++----- 1 file changed, 287 insertions(+), 89 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 87f155de0e65..a0f93756e717 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -45,9 +45,9 @@ enum { /* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ #define MLDSA87_SIGNATURE_SIZE 4627 -static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id, - __u32 fd_array_cnt) +static int load_loader_log(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt, char *log_buf, __u32 log_sz) { union bpf_attr attr; int fd; @@ -59,18 +59,31 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.license = ptr_to_u64("Dual BSD/GPL"); attr.prog_flags = BPF_F_SLEEPABLE; attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = fd_array_cnt; if (sig) { attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } - attr.fd_array_cnt = fd_array_cnt; + if (log_buf) { + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_sz; + } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); return fd < 0 ? -errno : fd; } +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + return load_loader_log(insns, insns_sz, map_fd, sig, sig_sz, keyring_id, + fd_array_cnt, NULL, 0); +} + static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, @@ -205,6 +218,7 @@ static int sign_buf_digest(const char *dir, const void *buf, __u32 len, fd = mkstemp(data_tmpl); if (fd < 0) return -errno; + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (write(fd, buf, len) != (ssize_t)len) { close(fd); ret = -EIO; @@ -229,30 +243,28 @@ static int sign_buf_digest(const char *dir, const void *buf, __u32 len, goto out; } - snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (stat(sigpath, &st) < 0) { ret = -errno; goto out; } if (st.st_size > (off_t)*sig_sz) { ret = -E2BIG; - goto out_sig; + goto out; } fd = open(sigpath, O_RDONLY); if (fd < 0) { ret = -errno; - goto out_sig; + goto out; } if (read(fd, sig, st.st_size) != st.st_size) { close(fd); ret = -EIO; - goto out_sig; + goto out; } close(fd); *sig_sz = st.st_size; -out_sig: - unlink(sigpath); out: + unlink(sigpath); unlink(data_tmpl); return ret; } @@ -564,7 +576,6 @@ static void signature_failure_logs(void) static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; char log_buf[1024] = {}; struct gen_loader_fixture f; - union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { @@ -573,22 +584,9 @@ static void signature_failure_logs(void) * failure is reported through the verifier log. A present-but- * invalid signature is rejected and the log says why. */ - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = ptr_to_u64(f.gopts.insns); - attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); - attr.license = ptr_to_u64("Dual BSD/GPL"); - attr.prog_flags = BPF_F_SLEEPABLE; - attr.signature = ptr_to_u64(junk); - attr.signature_size = sizeof(junk); - attr.keyring_id = KEY_SPEC_SESSION_KEYRING; - attr.log_level = 1; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = sizeof(log_buf); - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - - fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, - offsetofend(union bpf_attr, keyring_id)); + fd = load_loader_log(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0, + log_buf, sizeof(log_buf)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); if (fd >= 0) close(fd); @@ -708,6 +706,23 @@ static int bpf_keyring_lookup(int *nr_keys) return serial; } +static long keyctl_ret(int cmd, unsigned long arg2, unsigned long arg3) +{ + long ret = syscall(__NR_keyctl, cmd, arg2, arg3); + + return ret < 0 ? -errno : ret; +} + +/* + * What the bpf keyring still needs once it got provisioned: KEY_POS_SEARCH + * for the in-kernel search during verification, and the user view/read bits + * so it stays visible in /proc/keys, rest is dropped so the enrolled is + * therefore final. + */ +#define BPF_KEYRING_PERM_LOCKED 0x08030000 +/* What bpf_keyring_init() grants at boot. */ +#define BPF_KEYRING_PERM_INITIAL 0x082f0000 + static void bpf_keyring_sealed(void) { static const __u8 junk[64] = {}; @@ -721,6 +736,9 @@ static void bpf_keyring_sealed(void) } serial = bpf_keyring_lookup(NULL); if (serial >= 0) { + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, + KEY_SPEC_BPF_KEYRING, 0), serial, + "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, KEY_SPEC_BPF_KEYRING); if (key >= 0) @@ -739,6 +757,183 @@ static void bpf_keyring_sealed(void) gen_loader_fixture_fini(&f); } +static int try_load(const struct gen_loader_fixture *f, const void *sig, + __u32 sig_sz, __s32 keyring_id, char *log_buf, __u32 log_sz) +{ + int map_fd, prog_fd; + + map_fd = setup_meta_map(f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + return map_fd; + prog_fd = load_loader_log(f->gopts.insns, f->gopts.insns_sz, map_fd, + sig, sig_sz, keyring_id, 1, log_buf, log_sz); + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + return prog_fd; +} + +/* + * This needs bpf.keyring_unsealed=1 on the guest kernel command line, which + * vmtest.sh can pass via KERNEL_CMDLINE_EXTRA. There is no way to unseal the + * keyring from here, so without it the test skips. It also only works once + * per boot, as restricting a keyring cannot be undone. + */ +static void bpf_keyring_provisioned(void) +{ + char dir_tmpl[] = "/tmp/bpfkeyringXXXXXX"; + char bad_tmpl[] = "/tmp/bpfkeyringbadXXXXXX"; + __u8 *sig = NULL, *bad = NULL, *buf = NULL; + int serial, err; + int nr_keys = 0, der_fd = -1; + struct gen_loader_fixture f; + __u32 sig_sz = 8192, bad_sz; + bool have_fixture = false; + char *dir, *bad_dir = NULL; + char log_buf[1024] = {}; + char path[PATH_MAX]; + __u8 der[4096]; + ssize_t der_sz; + + serial = bpf_keyring_lookup(&nr_keys); + if (serial < 0) { + printf("%s:SKIP:no bpf keyring (needs CONFIG_KEYS)\n", __func__); + test__skip(); + return; + } + if (nr_keys != 0) { + printf("%s:SKIP:the bpf keyring has already been provisioned\n", + __func__); + test__skip(); + return; + } + + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("genkey", dir), "verify_sig_setup genkey")) + goto rmdir; + + snprintf(path, sizeof(path), "%s/signing_key.der", dir); + der_fd = open(path, O_RDONLY); + if (!ASSERT_OK_FD(der_fd, "open signing_key.der")) + goto rmdir; + der_sz = read(der_fd, der, sizeof(der)); + close(der_fd); + if (!ASSERT_GT(der_sz, 0, "read signing_key.der")) + goto rmdir; + + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + serial, "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + + err = syscall(__NR_add_key, "asymmetric", "", der, (size_t)der_sz, + KEY_SPEC_BPF_KEYRING); + if (err < 0 && errno == EPERM) { + printf("%s:SKIP:the bpf keyring is sealed, need bpf.keyring_unsealed=1\n", + __func__); + test__skip(); + goto rmdir; + } + if (!ASSERT_GE(err, 0, "add the signing key to the bpf keyring")) + goto rmdir; + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + -ENOKEY, "unrestricted keyring still not consulted"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused before provisioning"); + + if (!ASSERT_OK(syscall(__NR_keyctl, KEYCTL_RESTRICT_KEYRING, + KEY_SPEC_BPF_KEYRING, NULL, NULL), + "restrict bpf keyring")) + goto out; + + if (!ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, + NULL, 0), + "load signed by a key in the .bpf keyring")) + goto out; + + bad_dir = mkdtemp(bad_tmpl); + if (!ASSERT_OK_PTR(bad_dir, "mkdtemp unenrolled")) + goto out; + if (!ASSERT_OK(run_setup("genkey", bad_dir), "verify_sig_setup genkey unenrolled")) + goto out; + bad_sz = 8192; + bad = malloc(bad_sz); + if (!ASSERT_OK_PTR(bad, "bad sig buf")) + goto out; + if (!ASSERT_OK(sign_buf(bad_dir, buf, f.gopts.insns_sz + f.data_sz, bad, + &bad_sz), "sign with an unenrolled key")) + goto out; + + ASSERT_EQ(try_load(&f, bad, bad_sz, KEY_SPEC_BPF_KEYRING, log_buf, + sizeof(log_buf)), -ENOKEY, + "key outside the bpf keyring refused"); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "the bpf keyring was consulted"); + + f.blob[0] ^= 0xff; + err = try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0); + f.blob[0] ^= 0xff; + ASSERT_EQ(err, -EKEYREJECTED, "tampered metadata refused"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused once .bpf is in use"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -ENOENT, "keyring writable while the user bits are there"); + + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_LOCKED); + if (!ASSERT_OK(err, "drop the user bits on the bpf keyring")) + goto out; + + ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + "load still verified against the locked keyring"); + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + -EACCES, "the special id grants no rights of its own"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -EACCES, "unlink refused"); + err = keyctl_ret(KEYCTL_CLEAR, serial, 0); + ASSERT_EQ(err, -EACCES, "clear refused"); + err = keyctl_ret(KEYCTL_REVOKE, serial, 0); + ASSERT_EQ(err, -EACCES, "revoke refused"); + err = keyctl_ret(KEYCTL_INVALIDATE, serial, 0); + ASSERT_EQ(err, -EACCES, "invalidate refused"); + err = keyctl_ret(KEYCTL_SET_TIMEOUT, serial, 1); + ASSERT_EQ(err, -EACCES, "timeout refused"); + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_INITIAL); + ASSERT_EQ(err, -EACCES, "the bits cannot be granted back"); + + ASSERT_EQ(bpf_keyring_lookup(&nr_keys), serial, "keyring still there"); + ASSERT_EQ(nr_keys, 1, "the enrolled key survived"); +out: + if (have_fixture) + gen_loader_fixture_fini(&f); + genkey_dir_fini(bad_dir); + free(buf); + free(bad); + free(sig); +rmdir: + genkey_dir_fini(dir); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -1973,68 +2168,71 @@ cleanup: run_setup("cleanup", dir); } +enum subtest_boot { + BOOT_ANY, + BOOT_SEALED, + BOOT_UNSEALED, +}; + +static const struct { + const char *name; + void (*fn)(void); + enum subtest_boot boot; +} subtests[] = { + { "loadtime_no_map", loadtime_no_map, BOOT_SEALED }, + { "loadtime_with_map", loadtime_with_map, BOOT_SEALED }, + { "metadata_match", metadata_match, BOOT_ANY }, + { "signature_enforced", signature_enforced, BOOT_SEALED }, + { "signed_nonexcl_fd_array_rejected", signed_nonexcl_fd_array_rejected, BOOT_SEALED }, + { "signed_unfrozen_fd_array_rejected", signed_unfrozen_fd_array_rejected, BOOT_SEALED }, + { "signed_nonarray_fd_array_rejected", signed_nonarray_fd_array_rejected, BOOT_SEALED }, + { "signed_btf_fd_array_rejected", signed_btf_fd_array_rejected, BOOT_ANY }, + { "signed_module_kfunc_rejected", signed_module_kfunc_rejected, BOOT_SEALED }, + { "signature_failure_logs", signature_failure_logs, BOOT_SEALED }, + { "signature_too_large", signature_too_large, BOOT_ANY }, + { "signature_zero_size", signature_zero_size, BOOT_ANY }, + { "signature_bad_keyring", signature_bad_keyring, BOOT_SEALED }, + { "bpf_keyring_sealed", bpf_keyring_sealed, BOOT_ANY }, + { "mldsa_signed_load", mldsa_signed_load, BOOT_SEALED }, + { "metadata_ctx_max_entries_ignored", metadata_ctx_max_entries_ignored, BOOT_ANY }, + { "metadata_ctx_initial_value_ignored", metadata_ctx_initial_value_ignored, BOOT_ANY }, + { "signature_authenticates_insns", signature_authenticates_insns, BOOT_SEALED }, + { "signature_authenticates_metadata", signature_authenticates_metadata, BOOT_SEALED }, + { "hash_requires_frozen", hash_requires_frozen, BOOT_ANY }, + { "no_update_after_freeze", no_update_after_freeze, BOOT_ANY }, + { "freeze_writable_mmap", freeze_writable_mmap, BOOT_ANY }, + { "no_writable_mmap_frozen", no_writable_mmap_frozen, BOOT_ANY }, + { "map_hash_matches_libbpf", map_hash_matches_libbpf, BOOT_ANY }, + { "map_hash_multi_element", map_hash_multi_element, BOOT_ANY }, + { "map_hash_bad_size", map_hash_bad_size, BOOT_ANY }, + { "map_hash_unsupported_type", map_hash_unsupported_type, BOOT_ANY }, + { "lsm_signature_verdict", lsm_signature_verdict, BOOT_SEALED }, + { "signed_no_fd_array", signed_no_fd_array, BOOT_SEALED }, + { "signed_map_by_fd_rejected", signed_map_by_fd_rejected, BOOT_SEALED }, + { "signed_sparse_fd_array_rejected", signed_sparse_fd_array_rejected, BOOT_SEALED }, + { "bpf_keyring_provisioned", bpf_keyring_provisioned, BOOT_UNSEALED }, +}; + void test_signed_loader(void) { - if (test__start_subtest("loadtime_no_map")) - loadtime_no_map(); - if (test__start_subtest("loadtime_with_map")) - loadtime_with_map(); - if (test__start_subtest("metadata_match")) - metadata_match(); - if (test__start_subtest("signature_enforced")) - signature_enforced(); - if (test__start_subtest("signed_nonexcl_fd_array_rejected")) - signed_nonexcl_fd_array_rejected(); - if (test__start_subtest("signed_unfrozen_fd_array_rejected")) - signed_unfrozen_fd_array_rejected(); - if (test__start_subtest("signed_nonarray_fd_array_rejected")) - signed_nonarray_fd_array_rejected(); - if (test__start_subtest("signed_btf_fd_array_rejected")) - signed_btf_fd_array_rejected(); - if (test__start_subtest("signed_module_kfunc_rejected")) - signed_module_kfunc_rejected(); - if (test__start_subtest("signature_failure_logs")) - signature_failure_logs(); - if (test__start_subtest("signature_too_large")) - signature_too_large(); - if (test__start_subtest("signature_zero_size")) - signature_zero_size(); - if (test__start_subtest("signature_bad_keyring")) - signature_bad_keyring(); - if (test__start_subtest("bpf_keyring_sealed")) - bpf_keyring_sealed(); - if (test__start_subtest("mldsa_signed_load")) - mldsa_signed_load(); - if (test__start_subtest("metadata_ctx_max_entries_ignored")) - metadata_ctx_max_entries_ignored(); - if (test__start_subtest("metadata_ctx_initial_value_ignored")) - metadata_ctx_initial_value_ignored(); - if (test__start_subtest("signature_authenticates_insns")) - signature_authenticates_insns(); - if (test__start_subtest("signature_authenticates_metadata")) - signature_authenticates_metadata(); - if (test__start_subtest("hash_requires_frozen")) - hash_requires_frozen(); - if (test__start_subtest("no_update_after_freeze")) - no_update_after_freeze(); - if (test__start_subtest("freeze_writable_mmap")) - freeze_writable_mmap(); - if (test__start_subtest("no_writable_mmap_frozen")) - no_writable_mmap_frozen(); - if (test__start_subtest("map_hash_matches_libbpf")) - map_hash_matches_libbpf(); - if (test__start_subtest("map_hash_multi_element")) - map_hash_multi_element(); - if (test__start_subtest("map_hash_bad_size")) - map_hash_bad_size(); - if (test__start_subtest("map_hash_unsupported_type")) - map_hash_unsupported_type(); - if (test__start_subtest("lsm_signature_verdict")) - lsm_signature_verdict(); - if (test__start_subtest("signed_no_fd_array")) - signed_no_fd_array(); - if (test__start_subtest("signed_map_by_fd_rejected")) - signed_map_by_fd_rejected(); - if (test__start_subtest("signed_sparse_fd_array_rejected")) - signed_sparse_fd_array_rejected(); + bool unsealed = keyring_unsealed_boot(); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(subtests); i++) { + if (!test__start_subtest(subtests[i].name)) + continue; + if (subtests[i].boot == BOOT_SEALED && unsealed) { + printf("%s:SKIP:needs a boot without bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + if (subtests[i].boot == BOOT_UNSEALED && !unsealed) { + printf("%s:SKIP:needs bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + subtests[i].fn(); + } } From 8588fa592985accc22c6757d6397d588fe71752d Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:27 +0200 Subject: [PATCH 065/660] Documentation/bpf: Document the bpf keyring and improve examples Key generation is detailled for RSA and ML-DSA, the load example sets keyring_id to the bpf keyring with the session keyring shown only as the staging variant, and the LSM admission example anchors on the bpf keyring while allowlisting staged serials rather than treating a user keyring as ordinary trust. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-12-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/bpf/signing.rst | 283 +++++++++++++++++++++++++++++----- 1 file changed, 246 insertions(+), 37 deletions(-) diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst index e73eaaebd8b1..27fdad3df96f 100644 --- a/Documentation/bpf/signing.rst +++ b/Documentation/bpf/signing.rst @@ -254,21 +254,24 @@ returned. Only after the program has fully loaded, at the next hook (``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning: validly signed *and* fully verified. -A more realistic admission policy than "is it signed at all": accept programs -signed by a system keyring, accept a user-keyring signature only if the -key/keyring it was verified against is on an explicit allowlist, and emit a -tamper-evident record of every decision so that even denied attempts are -auditable. (Illustrative - error checking elided.) +A more realistic admission policy than "is it signed at all": base trust in +the bpf keyring or an explicit allowlist of a caller-supplied staging key/ +keyring, and emit a record of every decision including denied attempts. +(Illustrative - error checking elided.) .. code-block:: c - /* Serials of user keys/keyrings we additionally trust. */ + /* + * Serials of caller-supplied keyrings we are willing to stage. Empty + * on a system that has committed to the bpf keyring, where the kernel + * refuses them anyway. + */ struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, __s32); /* keyring_serial */ __type(value, __u8); __uint(max_entries, 64); - } trusted_user_keys SEC(".maps"); + } staging_keys SEC(".maps"); /* Audit stream consumed by a userspace logger. */ struct { @@ -291,11 +294,18 @@ auditable. (Illustrative - error checking elided.) if (kernel) return 0; /* trust in-kernel loads */ - if (verdict != BPF_SIG_VERIFIED) + if (verdict != BPF_SIG_VERIFIED) { ret = -EPERM; /* must be validly signed */ - else if (ktype == BPF_SIG_KEYRING_USER && - !bpf_map_lookup_elem(&trusted_user_keys, &serial)) - ret = -EPERM; /* key/keyring not allowlisted */ + } else switch (ktype) { + case BPF_SIG_KEYRING_BPF: + break; + case BPF_SIG_KEYRING_USER: + if (!bpf_map_lookup_elem(&staging_keys, &serial)) + ret = -EPERM; + break; + default: + ret = -EPERM; /* keyring not in policy */ + } d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0); if (d) { @@ -309,6 +319,11 @@ auditable. (Illustrative - error checking elided.) return ret; } +Such a policy is what makes a caller-supplied keyring usable at all on a system +that does not boot with ``bpf.keyring_unsealed=1``: the allowlist bounds which +staged keys count, and the LSM itself has to protect them from being tampered +with. + Observing a verified load: ``security_bpf_prog()`` -------------------------------------------------- @@ -381,8 +396,9 @@ that verdict covered all of its exclusive maps, rejecting any that did not - so a deny-by-default admission policy needs no second enforcement point. Use ``security_bpf_prog()`` to record or finally gate the verified programs once they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields -let a policy distinguish, for example, "verified and signed by a builtin key" -from "verified by a user key". A policy LSM such as IPE could consume the same +let a policy distinguish "verified against the operator's bpf keyring" from +"verified against a keyring the loader supplied itself", which is the +distinction that matters most. A policy LSM such as IPE could consume the same hooks to enforce system policy without writing any BPF, though none implements this today. @@ -390,33 +406,158 @@ Keyrings ======== ``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified -against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING`` -and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings; -any other value is treated as the serial of a user/session key or keyring. -The keyring is looked up first, before the signature bytes are examined, so a -signature naming a non-existent keyring is rejected up front, and a failed -verification aborts the load - so a program that loads successfully with a -signature always has consistent keyring fields recorded. +against. Four values are well-known; anything else is resolved as a caller- +supplied key or keyring, named either by its serial or by one of the other +``KEY_SPEC_*`` ids: + +.. list-table:: + :header-rows: 1 + + * - ``keyring_id`` + - Keyring + * - ``0`` + - builtin trusted keyring + * - ``VERIFY_USE_SECONDARY_KEYRING`` (``1``) + - secondary trusted keyring + * - ``VERIFY_USE_PLATFORM_KEYRING`` (``2``) + - platform keyring + * - ``KEY_SPEC_BPF_KEYRING`` (``-9``) + - the bpf keyring + * - anything else + - a caller-supplied user/session key or keyring + +The keyring is resolved first, before the signature bytes are examined, so a +signature naming a keyring that cannot be used is rejected up front, and a +failed verification aborts the load - a program that loads successfully with +a signature therefore always has consistent keyring fields recorded. + +The bpf keyring +--------------- + +A system keyring needs a kernel rebuild or a vouched-for enrollment to rotate a +key, and grants BPF-signing trust to keys trusted for everything else in the +kernel too. A caller-supplied keyring, at the other extreme, is filled by the +very process that loads the program and so carries no trust of its own. + +The bpf keyring fills that gap and is the trust anchor which a signed BPF +deployment should be built on top of: a keyring named ``.bpf``, selected with +``KEY_SPEC_BPF_KEYRING``, that an operator provisions at boot with a key +scoped to BPF program loading and nothing else in the kernel's trust hierarchy. +It is owned by the operator rather than by the loader, and rotatable across a +reboot without touching the kernel image. It is modelled after the dm-verity +keyring (see ``dm_verity.keyring_unsealed=``) and provisioned the same way: an +initrd runs the ``keyctl`` steps below before handing off to the rootfs. + +Provisioning +~~~~~~~~~~~~ + +The keyring is created during ``late_initcall`` and is **sealed empty** by +default: it carries a reject-all restriction, so no key can ever be added and +``KEY_SPEC_BPF_KEYRING`` fails with ``-ENOKEY`` for the whole boot. + +``bpf.keyring_unsealed=1`` leaves it unrestricted at init so the initrd can +provision it. The keyring is not linked into any process keyring, but the +``KEY_SPEC_BPF_KEYRING`` special key id addresses it directly, so its serial +does not have to be looked up first. Steps would be as follows:: + + keyctl padd asymmetric "" -9 < signing_key.der + keyctl restrict_keyring -9 + +Both steps are required: the keyring is consulted only once it is **non-empty +and restricted**. An unrestricted keyring is ignored even when it holds keys, +so a half-provisioned keyring is inert rather than a weaker trust anchor, and a +load naming it fails with ``-ENOKEY`` and a verifier log. Restricting cannot +be undone. + +More than one key is enrolled by repeating the ``keyctl padd`` step; the +restriction is applied once, after the last of them:: + + for key in /etc/bpf/keys/*.der; do + keyctl padd asymmetric "" -9 < $key + done + + keyctl restrict_keyring -9 + keyctl show -9 + +The restriction bounds what can be added, never what can be taken away. A key +that is already enrolled can still be unlinked, and the keyring cleared or +revoked, by anything running as root. That does not weaken the anchor, since +a keyring left empty is no longer consulted and a load naming it fails with +``-ENOKEY``, but it does take signed loading out until the next boot. Dropping +the user permissions the keyring no longer needs would close that; as a third +step in the initrd:: + + keyctl setperm -9 0x08030000 + +What remains is ``KEY_POS_SEARCH`` for the in-kernel search during verification, +plus ``KEY_USR_VIEW`` and ``KEY_USR_READ`` so the keyring stays visible in +``/proc/keys``. ``KEY_SPEC_BPF_KEYRING`` names the keyring but conveys no rights +of its own - the keyring is not possessed by anyone, so the ``KEY_POS_*`` bits +are only ever reached by the in-kernel search, and userspace is left with what +the ``KEY_USR_*`` bits grant. Once they are dropped, addressing it by the serial +``/proc/keys`` reports gets no further than the special id does. + +Provisioning has to complete before control passes to the rootfs. The keyring +takes any number of keys for as long as it is unrestricted, so it is the +restriction that bounds the enrolled set, not the first enrollment. Before +the initrd hands off control, it must therefore restrict the keyring after +the last enrollment. + +Enforcement +~~~~~~~~~~~ + +``bpf.keyring_unsealed=1`` states that the bpf keyring is *the* trust anchor for +this boot, so it does more than unseal. From the first program load onwards a +caller-supplied user/session keyring is refused with ``-EPERM`` and a verifier +log message, whether or not provisioning ever completed. The system keyrings +stay selectable. + +Enforcement is readable at ``/sys/module/bpf/parameters/keyring_unsealed``, and +read-only there: the flag is ``__ro_after_init`` behind a 0444 parameter. It is +also derived from the boot flag rather than from the keyring's runtime state, +so there is no window early in boot during which a caller-supplied keyring is +still accepted. + +Caller-supplied keyrings are for staging +---------------------------------------- + +A ``keyring_id`` naming a user or session key or keyring is a *staging* +mechanism, not a trust anchor: it is filled by the same userspace that loads the +program, so verifying against it establishes only that the loader signed what it +loaded. Its purpose is to let software installed onto a running system - whose +signing key is not enrolled anywhere yet - run signed until that key reaches the +bpf keyring on the next boot. + +A system that has committed to the bpf keyring refuses this path outright (see +`Enforcement`_). A system that has not can still allow it, but a policy must +never treat ``BPF_SIG_KEYRING_USER`` as equivalent to the bpf or system +keyrings; it should allowlist the specific serials it is willing to stage and +pair that with a BPF LSM policy protecting those keys from tampering, as in +`Enforcement via LSMs`_. + +Recorded fields +--------------- Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: ``keyring_type`` (``enum bpf_sig_keyring``) Classified purely from ``keyring_id`` whenever the program is signed: ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system - keyrings, or ``_USER`` for a user/session keyring. It is - ``BPF_SIG_KEYRING_NONE`` for an unsigned program. + keyrings, ``_BPF`` for the bpf keyring, or ``_USER`` for a caller-supplied + user/session keyring. It is ``BPF_SIG_KEYRING_NONE`` for an unsigned + program. ``keyring_serial`` (``s32``) Set **only** on a successful verification, to the serial of the - **user/session key or keyring** that ``keyring_id`` resolved to - the + **caller-supplied key or keyring** that ``keyring_id`` resolved to - the object the signature was verified against, not the individual asymmetric key inside it that matched the signer. Passing ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's - serial. The system keyrings are trusted as a whole and expose no serial - here, so the serial is ``0`` for builtin, secondary and platform - signatures, and ``0`` for unsigned programs. In other words, a non-zero - ``keyring_serial`` is exactly "verified against the user key/keyring with - this serial". + serial. The system keyrings and the bpf keyring are trusted as a whole and + expose no serial here, so the serial is ``0`` for them, and ``0`` for + unsigned programs. A non-zero ``keyring_serial`` is therefore exactly + "verified against the caller-supplied key/keyring with this serial", which + is exactly the case a policy has to scrutinise. .. list-table:: :header-rows: 1 @@ -436,16 +577,51 @@ Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: * - ``VERIFY_USE_PLATFORM_KEYRING`` - ``BPF_SIG_KEYRING_PLATFORM`` - ``0`` - * - other (a user/session key serial) + * - ``KEY_SPEC_BPF_KEYRING`` + - ``BPF_SIG_KEYRING_BPF`` + - ``0`` + * - other (a caller-supplied key serial) - ``BPF_SIG_KEYRING_USER`` - serial of the resolved key/keyring -Producing a signed object -========================== +Producing and loading a signed object +===================================== + +Generating a signing key +------------------------ + +Signing is algorithm agnostic: the algorithm comes from the X.509 certificate +and the PKCS#7 ``SignerInfo``. Anything the X.509 and PKCS#7 parsers understand +works with no BPF-side change. Both examples below read the certificate request +config from ``x509.genkey``, for which ``certs/x509.genkey`` serves as a +template (see Documentation/admin-guide/module-signing.rst). RSA:: + + openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ + -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +ML-DSA-87 (FIPS-204), which needs openssl 3.5 or later, and both +``CONFIG_CRYPTO_MLDSA`` and ``CONFIG_CRYPTO_SHA512`` in the kernel - the latter +for the signedAttrs digest described below, which ``CONFIG_CRYPTO_MLDSA`` does +not select. Note the absence of a digest option: ML-DSA hashes the message +itself, so openssl ignores an explicit digest for it:: + + openssl req -new -nodes -utf8 -days 36500 -batch -x509 \ + -newkey ML-DSA-87 -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +``bpftool`` handles the following internally: openssl before 4.0 cannot combine +ML-DSA with ``CMS_NOATTR``, so it falls back to signedAttrs, where only SHA-512 +is permitted. This mirrors what module signing does as well. + +Signing +------- ``bpftool`` generates and signs a light skeleton in one step:: - bpftool gen skeleton -L -S -k -i \ + bpftool gen skeleton -L -S -k signing_key.pem -i signing_key.der \ obj.bpf.o > obj.lskel.h ``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables @@ -454,12 +630,36 @@ signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate. reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader instructions so the metadata map can be bound to the loader. The signature and hash are embedded in the generated header; the certificate is used only for -signing and is not included. Loading the skeleton performs the -create/populate/freeze/load sequence described above. +signing and is not included. -At runtime the trusted public key must be present in the chosen keyring (for -example added to the session keyring, or built into the kernel's builtin trusted -keyring) for verification to succeed. +Loading +------- + +The generated skeleton exposes ``keyring_id``, which selects the keyring the +kernel verifies against. Set it between open and load; loading then performs +the create/populate/freeze/load sequence described above:: + + struct obj *skel = obj__open(); + + skel->keyring_id = KEY_SPEC_BPF_KEYRING; + err = obj__load(skel); + +For the staging case the same object is loaded against a keyring the caller +populated itself, which only works on a system that has not set +``bpf.keyring_unsealed=1``:: + + /* + * Staging only: this keyring is under the loader's own control and + * carries no trust of its own. See "Caller-supplied keyrings are for + * staging". + */ + key_id = add_key("asymmetric", "", der, der_sz, KEY_SPEC_SESSION_KEYRING); + skel->keyring_id = KEY_SPEC_SESSION_KEYRING; + err = obj__load(skel); + +Either way the trusted public key must already be in the chosen keyring for +verification to succeed. For the bpf keyring that enrollment happens once at +boot, see `Provisioning`_. UAPI reference ============== @@ -487,6 +687,13 @@ UAPI reference The map content is not hashed separately at all - it is covered, as bytes, by the program signature. +Kernel command line: + +``bpf.keyring_unsealed=`` + Set to ``1`` to leave the bpf keyring unsealed for provisioning, and to make + it the only non-system keyring a loader may select for the rest of the boot + (see `The bpf keyring`_). + Notes and limitations ====================== @@ -495,3 +702,5 @@ Notes and limitations exceed it. - The metadata container is a single-element array map, accessed through ``map_direct_value_addr``. +- The bpf keyring needs ``CONFIG_KEYS``; without it there is no bpf keyring + and ``KEY_SPEC_BPF_KEYRING`` never resolves. From bd71904697c64fe88e83322a79f8e6e68d0306d1 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:02 -0700 Subject: [PATCH 066/660] selftests/bpf: Add compare_text_to_expected() helper Some selftests generate text (such as BTF dump) and check it against an expectation committed nearby. There is no shared way to do that. prog_tests/btf_dump.c assembles an "awk ... | diff -u" pipeline and hands it to system(). Any other test wanting the same behaviour needs to reproduce both the comparison and the reporting of a mismatch. test_progs captures per-subtest output by pointing the stdout and stderr FILE * globals at a memstream. But a child process inherits descriptors, not the globals. So the "| diff -u" goes to the console instead of the subtest log and is absent from the failure report. Add a helper that compares two strings and runs diff(1) on mismatch, properly relaying the output to stdout. Add tests for the helper. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-2-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../bpf/prog_tests/prog_tests_framework.c | 23 ++++++++++ tools/testing/selftests/bpf/testing_helpers.c | 43 +++++++++++++++++++ tools/testing/selftests/bpf/testing_helpers.h | 3 ++ 3 files changed, 69 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index 7607cfc2408c..d111fe105447 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -179,3 +179,26 @@ void test_prog_tests_framework_expected_msgs(void) } } } + +void test_prog_tests_framework_compare_text(void) +{ + int err; + + if (test__start_subtest("compare_text_match")) { + err = compare_text_to_expected("same\n", "same\n"); + ASSERT_EQ(err, 0, "match_rc"); + test__end_subtest(); + } + + if (test__start_subtest("compare_text_mismatch")) { + err = compare_text_to_expected("line two\n", "line one\n"); + fflush(stdout); + + ASSERT_EQ(err, -1, "mismatch_rc"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", + "diff_has_expected"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", + "diff_has_actual"); + test__end_subtest(); + } +} diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index c970e7793dfc..3f037949e978 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -534,3 +534,46 @@ int stack_mprotect(void) PROT_READ | PROT_WRITE | PROT_EXEC); return ret; } + +int compare_text_to_expected(const char *actual, const char *expected) +{ + char exp_path[] = "/tmp/selftest_expected.XXXXXX"; + char act_path[] = "/tmp/selftest_actual.XXXXXX"; + char buf[512], cmd[128]; + int exp_fd, act_fd; + FILE *p; + + if (!strcmp(actual, expected)) + return 0; + + exp_fd = mkstemp(exp_path); + act_fd = mkstemp(act_path); + if (exp_fd < 0 || act_fd < 0) { + fprintf(stdout, "output differs, no temp file for a diff\n"); + goto out; + } + + dprintf(exp_fd, "%s", expected); + dprintf(act_fd, "%s", actual); + + snprintf(cmd, sizeof(cmd), "diff -u '%s' '%s'", exp_path, act_path); + p = popen(cmd, "r"); + if (!p) { + fprintf(stdout, "output differs, '%s' did not run\n", cmd); + goto out; + } + while (fgets(buf, sizeof(buf), p)) + fputs(buf, stdout); + pclose(p); + +out: + if (exp_fd >= 0) { + close(exp_fd); + unlink(exp_path); + } + if (act_fd >= 0) { + close(act_fd); + unlink(act_path); + } + return -1; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 2edc6fb7fc52..1c58a2f08b64 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -61,4 +61,7 @@ int testing_prog_flags(void); bool is_jit_enabled(void); int stack_mprotect(void); +/* Runs diff(1) on mismatch */ +int compare_text_to_expected(const char *actual, const char *expected); + #endif /* __TESTING_HELPERS_H */ From c0722dc3288aa5f53e058c91bebd100bcc6682d0 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:03 -0700 Subject: [PATCH 067/660] selftests/bpf: Modernize btf_dump test scaffolding Refactor test_btf_dump_case() in order to: * use newer ASSERT_* macros instead of CHECK * drop the file-scope "duration" variable CHECK required Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-3-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/btf_dump.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 9f1b50e07a29..e9d2d4c26509 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -2,8 +2,6 @@ #include #include -static int duration = 0; - void btf_dump_printf(void *ctx, const char *fmt, va_list args) { vfprintf(ctx, fmt, args); @@ -69,8 +67,9 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) if (!t->known_ptr_sz) { btf__set_pointer_size(btf, 8); } else { - CHECK(btf__pointer_size(btf) != 8, "ptr_sz", "exp %d, got %zu\n", - 8, btf__pointer_size(btf)); + size_t ptr_sz = btf__pointer_size(btf); + + ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); @@ -80,8 +79,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) goto done; } f = fdopen(fd, "w"); - if (CHECK(f == NULL, "open_tmp", "failed to open file: %s(%d)\n", - strerror(errno), errno)) { + if (!ASSERT_OK_PTR(f, "open_tmp")) { close(fd); goto done; } @@ -89,9 +87,8 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) err = btf_dump_all_types(btf, f); fclose(f); close(fd); - if (CHECK(err, "btf_dump", "failure during C dumping: %d\n", err)) { + if (!ASSERT_OK(err, "btf_dump")) goto done; - } snprintf(test_file, sizeof(test_file), "progs/%s.c", t->file); if (access(test_file, R_OK) == -1) @@ -114,10 +111,10 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", test_file, out_file); err = system(diff_cmd); - if (CHECK(err, "diff", - "differing test output, output=%s, err=%d, diff cmd:\n%s\n", - out_file, err, diff_cmd)) + if (!ASSERT_OK(err, "diff")) { + fprintf(stdout, "output=%s, diff cmd:\n%s\n", out_file, diff_cmd); goto done; + } remove(out_file); From 122255fb1933a57d81d05d0224d417969ec35fa9 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:04 -0700 Subject: [PATCH 068/660] selftests/bpf: Compare btf_dump expected output in-process test_btf_dump_case() writes the dumped types to a temporary file and then runs a command against the expectation: system("awk '...' 'progs/x.c' | diff -u - '/tmp/x.output.XXXXXX'"); This creates three processes per test case: a shell, awk and diff. The awk cannot be deferred to the failure path, because its output is the expected text. In a VM with a 9p root (used on BPF CI) process creation dominates the cost. A bare fork+exec measures around 110ms, so the btf_dump test cases spend most of their runtime forking. Replace the awk pipeline with a simple marker parser. Buffer the dump in memory, and use compare_text_to_expected(), which only runs diff(1) on mismatch. Measured with "time ./test_progs -t btf_dump" in the VM: real 2.751s -> 0.702s, sys 1.901s -> 0.193s This also drops the temporary file setup. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-4-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/btf_dump.c | 112 +++++++++++++----- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index e9d2d4c26509..fe04a955d46c 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include "testing_helpers.h" void btf_dump_printf(void *ctx, const char *fmt, va_list args) { @@ -43,12 +44,75 @@ done: return err; } +/* + * Expected output is embedded in the test case source, between + * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT markers. A region is either + * plain C, where a declaration doubles as its own expectation, or C wrapped in + * a block comment, where the rendered form differs from the source. All + * regions of a file concatenate into one expectation, compared against one + * whole-file dump. + * + * Returns a malloc'd buffer for the caller to free, or NULL on failure. + */ +static char *read_expected_output(const char *path) +{ + size_t out_sz = 0, line_cap = 0; + char *out = NULL, *line = NULL; + bool in_region = false; + FILE *f, *out_file; + + f = fopen(path, "r"); + if (!f) + return NULL; + + out_file = open_memstream(&out, &out_sz); + if (!out_file) { + fclose(f); + return NULL; + } + + while (getline(&line, &line_cap, f) > 0) { + const char *p; + + if (strstr(line, "START-EXPECTED-OUTPUT")) { + in_region = true; + continue; + } + if (strstr(line, "END-EXPECTED-OUTPUT")) + in_region = false; + if (!in_region) + continue; + + p = line + strspn(line, " \t"); + + /* opening or closing line of a commented out region */ + if (!strncmp(p, "/*", 2) || !strncmp(p, "*/", 2)) + continue; + + /* + * Only a '*' directly after the indentation is a comment + * prefix. Without one the line is taken as it is, leading + * whitespace included. + */ + p = *p == '*' ? p + 1 : line; + + fputs(p, out_file); + } + + free(line); + fclose(f); + fclose(out_file); + return out; +} + static int test_btf_dump_case(int n, struct btf_dump_test_case *t) { - char test_file[256], out_file[256], diff_cmd[1024]; + char *dump = NULL, *expected = NULL; struct btf *btf = NULL; - int err = 0, fd = -1; - FILE *f = NULL; + char test_file[256]; + size_t dump_sz = 0; + int err = 0; + FILE *f; snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file); @@ -72,21 +136,14 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } - snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); - fd = mkstemp(out_file); - if (!ASSERT_GE(fd, 0, "create_tmp")) { - err = fd; - goto done; - } - f = fdopen(fd, "w"); - if (!ASSERT_OK_PTR(f, "open_tmp")) { - close(fd); + f = open_memstream(&dump, &dump_sz); + if (!ASSERT_OK_PTR(f, "open_memstream")) { + err = -errno; goto done; } err = btf_dump_all_types(btf, f); fclose(f); - close(fd); if (!ASSERT_OK(err, "btf_dump")) goto done; @@ -97,28 +154,23 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) * without preserving the directory structure. */ snprintf(test_file, sizeof(test_file), "%s.c", t->file); - /* - * Diff test output and expected test output, contained between - * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT lines in test case. - * For expected output lines, everything before '*' is stripped out. - * Also lines containing comment start and comment end markers are - * ignored. - */ - snprintf(diff_cmd, sizeof(diff_cmd), - "awk '/START-EXPECTED-OUTPUT/{out=1;next} " - "/END-EXPECTED-OUTPUT/{out=0} " - "/\\/\\*|\\*\\//{next} " /* ignore comment start/end lines */ - "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", - test_file, out_file); - err = system(diff_cmd); - if (!ASSERT_OK(err, "diff")) { - fprintf(stdout, "output=%s, diff cmd:\n%s\n", out_file, diff_cmd); + + expected = read_expected_output(test_file); + if (!ASSERT_OK_PTR(expected, "read_expected_output")) { + err = -errno; goto done; } - remove(out_file); + /* + * The mismatch has already been reported, so this only has to + * register the failure. ASSERT_OK() would append a stale errno to it. + */ + err = compare_text_to_expected(dump, expected); + ASSERT_EQ(err, 0, "compare_text_to_expected"); done: + free(expected); + free(dump); btf__free(btf); return err; } From 4f8a4c76f6704667111d76404bdea4fef0373358 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:05 -0700 Subject: [PATCH 069/660] selftests/bpf: NUL-terminate bpftool command output run_command() writes a bpftool command's output into a caller-supplied buffer, and every caller treats that buffer as a C string. However fread() reports a byte count and doesn't terminate the string. The helper does not terminate either, so callers have to zero the buffer first. prog_tests/bpftool_metadata.c does not, for example. Read one byte less and terminate in the helper. Fixes: f21fae577446 ("selftests/bpf: Add a few helpers for bpftool testing") Acked-by: Eduard Zingerman Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-5-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/bpftool_helpers.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index 0a2a4f0a2794..c49fdd90eb03 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c @@ -53,6 +53,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) static char bpftool_path[PATH_MAX] = {}; bool suppress_output = !(output_buf && output_max_len); char command[BPFTOOL_FULL_CMD_MAX_LEN]; + size_t n; FILE *f; int ret; @@ -68,8 +69,10 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) if (!f) return 1; - if (!suppress_output) - fread(output_buf, 1, output_max_len, f); + if (!suppress_output) { + n = fread(output_buf, 1, output_max_len - 1, f); + output_buf[n] = '\0'; + } ret = pclose(f); return ret; From 438147f7f60104a228fb506352bc8bbae53b1efd Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:06 -0700 Subject: [PATCH 070/660] selftests/bpf: Add tests for bpftool btf dump format c "bpftool btf dump format c" generates the vmlinux.h that BPF programs are built against, and it has no test coverage at all. The only in-tree consumers are build systems. Building against the header only catches what a compiler rejects, which is insufficient. Introduce a bpftool_btf_dump selftest. Dump a small hand-built BTF and compare the output against committed expectations. The BTF is assembled with btf__add_*() rather than compiled from BPF C because the fixture needs a 4-byte "long int". That is what makes bpftool render the hole in struct holey as a pair of "long: 32;" bitfields. A BPF target is always 64-bit, so a compiled fixture could not supply it. Comparing the whole dump makes every change to a generated header show up in a patch. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-6-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/Makefile | 8 +- .../bpf/bpftool_btf_dump_sorted.expected | 44 +++++ .../bpf/bpftool_btf_dump_unsorted.expected | 48 +++++ .../bpf/prog_tests/bpftool_btf_dump.c | 178 ++++++++++++++++++ 4 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 05b3ee64290f..411a13681155 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -119,7 +119,9 @@ TEST_GEN_PROGS += test_progs-cpuv4 TEST_INST_SUBDIRS += cpuv4 endif -TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) +TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ @@ -887,7 +889,9 @@ TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ ima_setup.sh \ $(VERIFY_SIG_SETUP) \ $(wildcard progs/btf_dump_test_case_*.c) \ - $(wildcard progs/*.bpf.o) + $(wildcard progs/*.bpf.o) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS $(eval $(call DEFINE_TEST_RUNNER,test_progs)) diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected new file mode 100644 index 000000000000..a9c6b688ffd7 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected @@ -0,0 +1,44 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected new file mode 100644 index 000000000000..4310adc2ed51 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected @@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +enum { + E0 = 1, +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c new file mode 100644 index 000000000000..d5b25302b0c8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "btf_helpers.h" +#include "testing_helpers.h" + +#define DUMP_BUF_SZ (16 * 1024) + +#define EXPECTED_SORTED "bpftool_btf_dump_sorted.expected" +#define EXPECTED_UNSORTED "bpftool_btf_dump_unsorted.expected" + +/* + * struct holey { + * int c; + * <64-bit hole> + * int tail; + * }; + * enum { E0 = 1 }; + * struct s { int f; }; + */ +static struct btf *mk_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_int(btf, "long int", 4, BTF_INT_SIGNED); + + btf__add_struct(btf, "holey", 16); + btf__add_field(btf, "c", 1, 0, 0); + btf__add_field(btf, "tail", 1, 96, 0); + + btf__add_enum(btf, NULL, 4); + btf__add_enum_value(btf, "E0", 1); + + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'long int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[3] STRUCT 'holey' size=16 vlen=2\n" + "\t'c' type_id=1 bits_offset=0\n" + "\t'tail' type_id=1 bits_offset=96", + "[4] ENUM '(anon)' encoding=UNSIGNED size=4 vlen=1\n" + "\t'E0' val=1", + "[5] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0"); + + return btf; +} + +static int btf_to_tmpfile(const struct btf *btf, char *path) +{ + ssize_t written; + const void *raw; + __u32 sz; + int fd; + + raw = btf__raw_data(btf, &sz); + if (!ASSERT_OK_PTR(raw, "raw_data")) + return -1; + + snprintf(path, PATH_MAX, "/tmp/bpftool_btf_dump.XXXXXX"); + fd = mkstemp(path); + if (!ASSERT_OK_FD(fd, "mkstemp_btf")) + return -1; + + written = write(fd, raw, sz); + close(fd); + if (!ASSERT_EQ(written, sz, "write_btf")) { + unlink(path); + return -1; + } + + return 0; +} + +static char *dump_c(const char *btf_path, bool sorted) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s format c%s", + btf_path, sorted ? "" : " unsorted"); + + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_format_c")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *read_expected(const char *path) +{ + char *buf = NULL; + size_t cap = 0; + FILE *f; + int err; + + f = fopen(path, "r"); + if (!f) { + err = errno; + PRINT_FAIL("can't open expected output '%s': errno %d\n", path, err); + return NULL; + } + + /* no NUL in a generated header, so this reads to the end */ + if (getdelim(&buf, &cap, '\0', f) < 0) { + err = errno; + PRINT_FAIL("can't read expected output '%s': errno %d\n", path, err); + free(buf); + buf = NULL; + } + + fclose(f); + return buf; +} + +static void test_dump(const char *btf_path, bool sorted) +{ + const char *exp_path; + char *dump, *exp; + int err; + + exp_path = sorted ? EXPECTED_SORTED : EXPECTED_UNSORTED; + + dump = dump_c(btf_path, sorted); + if (!dump) + return; + + exp = read_expected(exp_path); + if (!exp) + goto out_dump; + + err = compare_text_to_expected(dump, exp); + ASSERT_OK(err, sorted ? "cmp_sorted" : "cmp_unsorted"); + + free(exp); +out_dump: + free(dump); +} + +void test_bpftool_btf_dump(void) +{ + char path[PATH_MAX]; + struct btf *btf; + + btf = mk_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("c_sorted")) + test_dump(path, true); + if (test__start_subtest("c_unsorted")) + test_dump(path, false); + + unlink(path); +out_btf: + btf__free(btf); +} From ff01f88a91368c264efb071c13f2b27b4d5c1d2b Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:07 -0700 Subject: [PATCH 071/660] bpftool: Don't drop a type in the sorted C dump The C dump sorts types by default, so that generated headers are diffable. The sorted dump emits one type fewer than the unsorted dump of the same BTF. dump_btf_c() starts its loop at index 1 to skip the void type at BTF type ID 0. That holds for the unsorted dump, where the array index is the type ID, but not after qsort(): position 0 is then the lowest ranked type, and btf_type_rank() ranks an anonymous enum 0 while void takes the default rank of 10. So the enum is skipped, and void is emitted instead as a no-op. Fixes: 94133cf24bb3 ("bpftool: Introduce btf c dump sorting") Acked-by: Eduard Zingerman Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-7-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/btf.c | 6 +++++- .../testing/selftests/bpf/bpftool_btf_dump_sorted.expected | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index c9589026da8d..bca0a3982f09 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -805,9 +805,13 @@ static int dump_btf_c(const struct btf *btf, if (sort_dump) datums = sort_btf_c(btf); - for (i = 1; i < cnt; i++) { + for (i = 0; i < cnt; i++) { int idx = datums ? datums[i].index : i; + /* type ID 0 is void, skip it */ + if (idx == 0) + continue; + err = btf_dump__dump_type(d, idx); if (err) goto done; diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected index a9c6b688ffd7..5470b2b43229 100644 --- a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected @@ -21,6 +21,10 @@ #endif #endif +enum { + E0 = 1, +}; + struct holey { int c; long: 32; From d761934c9483ecde93fe99d8705282f716dfee50 Mon Sep 17 00:00:00 2001 From: Sean Young Date: Thu, 20 Aug 2026 22:43:29 +0100 Subject: [PATCH 072/660] selftests/bpf: Convert lirc_mode2 to prog_tests and extend coverage While porting, extend coverage of the BPF_LIRC_MODE2 attach/detach/ query API: - bpf_prog_attach() with invalid flags is rejected with -EINVAL and does not attach the program - bpf_prog_query() with invalid flags is rejected with -EINVAL without disturbing existing attachments - bpf_prog_query() reports the correct program id, not just count, at each step, via bpf_prog_get_info_by_fd() - a lirc chardev can hold more than one attached program: load a second, independent instance, attach it alongside the first, confirm both are reported by bpf_prog_query(), then detach it without disturbing the first program's attachment - detaching an already-detached program consistently fails with -ENOENT, for both the first and second program Signed-off-by: Sean Young Assisted-by: Claude:claude-sonnet-5 Link: https://lore.kernel.org/bpf/20260820214332.89030-1-sean@mess.org Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/.gitignore | 1 - tools/testing/selftests/bpf/Makefile | 3 - .../selftests/bpf/prog_tests/lirc_mode2.c | 334 ++++++++++++++++++ .../testing/selftests/bpf/progs/lirc_mode2.c | 32 ++ .../bpf/progs/test_lirc_mode2_kern.c | 26 -- .../testing/selftests/bpf/test_lirc_mode2.sh | 41 --- .../selftests/bpf/test_lirc_mode2_user.c | 177 ---------- 7 files changed, 366 insertions(+), 248 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/lirc_mode2.c create mode 100644 tools/testing/selftests/bpf/progs/lirc_mode2.c delete mode 100644 tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c delete mode 100755 tools/testing/selftests/bpf/test_lirc_mode2.sh delete mode 100644 tools/testing/selftests/bpf/test_lirc_mode2_user.c diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 986a6389186b..b815bf0d8877 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -17,7 +17,6 @@ test_verifier_log feature urandom_read test_sockmap -test_lirc_mode2_user flow_dissector_load test_tcpnotify_user test_libbpf diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 411a13681155..b481b867f372 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -125,7 +125,6 @@ TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ - test_lirc_mode2.sh \ test_bpftool_build.sh \ test_doc_build.sh \ test_xsk.sh \ @@ -143,7 +142,6 @@ TEST_GEN_PROGS_EXTENDED = \ bench \ flow_dissector_load \ test_cpp \ - test_lirc_mode2_user \ veristat \ xdp_features \ xdp_hw_metadata \ @@ -340,7 +338,6 @@ $(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS) $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) -$(OUTPUT)/test_lirc_mode2_user: $(TESTING_HELPERS) $(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) $(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) diff --git a/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c new file mode 100644 index 000000000000..7b7352dbcd82 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young + +// A lirc chardev is a device representing a consumer IR (cir) device which +// can receive infrared signals from remote control and/or transmit IR. +// +// IR is sent as a series of pulses and space somewhat like morse code. The +// BPF program can decode this into scancodes so that rc-core can translate +// this into input key codes using the rc keymap. +// +// This test works by sending IR over rc-loopback, so the IR is processed by +// BPF and then decoded into scancodes. The lirc chardev must be the one +// associated with rc-loopback, see the output of ir-keytable(1). +// +// The following CONFIG options must be enabled for the test to succeed: +// CONFIG_RC_CORE=y +// CONFIG_BPF_LIRC_MODE2=y +// CONFIG_RC_LOOPBACK=y +// CONFIG_LIRC=y + +#include +#include +#include +#include +#include +#include +#include "lirc_mode2.skel.h" + +/* + * Read the DEVNAME= line out of the first uevent file that matches + * pattern, and turn it into a /dev/ path. + */ +static bool find_devname(const char *pattern, char *path, size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + FILE *f; + + if (glob(pattern, 0, NULL, &gl) || gl.gl_pathc == 0) + goto out; + + f = fopen(gl.gl_pathv[0], "r"); + if (!f) + goto out; + + char line[256]; + + while (fgets(line, sizeof(line), f)) { + char *val; + + if (strncmp(line, "DEVNAME=", 8)) + continue; + + val = line + 8; + val[strcspn(val, "\n")] = '\0'; + snprintf(path, path_sz, "/dev/%s", val); + found = true; + break; + } + + fclose(f); +out: + globfree(&gl); + return found; +} + +/* Load rc-loopback and find the lirc and input chardevs it created. */ +static bool find_loopback_devices(char *lirc_path, char *input_path, + size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + + /* Ignore failure, we check for the resulting devices below. */ + SYS_NOFAIL("modprobe rc-loopback > /dev/null 2>&1"); + + if (glob("/sys/class/rc/rc*", 0, NULL, &gl)) + goto out; + + for (size_t i = 0; i < gl.gl_pathc; i++) { + const char *rcdir = gl.gl_pathv[i]; + char uevent_path[PATH_MAX]; + char uevent[4096]; + char pattern[PATH_MAX]; + FILE *f; + size_t n; + + snprintf(uevent_path, sizeof(uevent_path), "%s/uevent", rcdir); + f = fopen(uevent_path, "r"); + if (!f) + continue; + n = fread(uevent, 1, sizeof(uevent) - 1, f); + fclose(f); + uevent[n] = '\0'; + + if (!strstr(uevent, "DRV_NAME=rc-loopback")) + continue; + + snprintf(pattern, sizeof(pattern), "%s/lirc*/uevent", rcdir); + if (!find_devname(pattern, lirc_path, path_sz)) + continue; + + snprintf(pattern, sizeof(pattern), "%s/input*/event*/uevent", rcdir); + if (!find_devname(pattern, input_path, path_sz)) + continue; + + found = true; + break; + } + +out: + if (!found) { + fprintf(stderr, "No rc devices found\n"); + fprintf(stderr, "Enable CONFIG_RC_LOOPBACK and CONFIG_BPF_LIRC_MODE2\n"); + } + + globfree(&gl); + return found; +} + +void test_lirc_mode2(void) +{ + char lirc_path[PATH_MAX], input_path[PATH_MAX]; + int lircfd = -1, inputfd = -1, progfd, progfd2 = -1; + struct lirc_mode2 *skel = NULL, *skel2 = NULL; + __u32 prog_ids[10], prog_flags[10], prog_cnt; + struct bpf_prog_info info; + __u32 info_len, prog_id, prog_id2; + int testir1 = 0x8ead; /* keydown flag (0x8000) | scancode 0xead */ + int testir2 = 0x4081; /* pointer_rel flag (0x4000) | rel_x=1 | rel_y=1 */ + struct input_event event; + struct pollfd pfd = {}; + int ret; + + if (getuid() != 0) { + test__skip(); + return; + } + + if (!find_loopback_devices(lirc_path, input_path, sizeof(lirc_path))) { + test__skip(); + return; + } + + skel = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel, "lirc_mode2__open_and_load")) + return; + + progfd = bpf_program__fd(skel->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd, &info, &info_len); + if (!ASSERT_OK(ret, "get first program's info")) + goto out; + prog_id = info.id; + + lircfd = open(lirc_path, O_RDWR | O_NONBLOCK); + if (!ASSERT_GE(lircfd, 0, "open lirc device")) + goto out; + + /* Try to detach it before it was ever attached, should fail. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_EQ(ret, -ENOENT, "detach unattached program")) + goto out; + + inputfd = open(input_path, O_RDONLY | O_NONBLOCK); + if (!ASSERT_GE(inputfd, 0, "open input device")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs before attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "no programs should be attached yet")) + goto out; + + /* Invalid attach flags must be rejected, and must not attach. */ + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 1); + if (!ASSERT_EQ(ret, -EINVAL, "attach with invalid flags")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after rejected attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "rejected attach should not attach")) + goto out; + + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach program to lirc device")) + goto out; + + /* Invalid query flags must be rejected too, without upsetting state. */ + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 1, prog_flags, prog_ids, + &prog_cnt); + ASSERT_EQ(ret, -EINVAL, "query with invalid flags"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after attach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should be attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "queried id should match attached program"); + + /* Write raw IR */ + ret = write(lircfd, &testir1, sizeof(testir1)); + if (!ASSERT_EQ(ret, sizeof(testir1), "send test IR message 1")) + goto out_detach; + + pfd.fd = inputfd; + pfd.events = POLLIN; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 1")) + goto out_detach; + + if (event.type == EV_MSC && event.code == MSC_SCAN && + event.value == 0xead) + break; + } + + /* Write raw IR */ + ret = write(lircfd, &testir2, sizeof(testir2)); + if (!ASSERT_EQ(ret, sizeof(testir2), "send test IR message 2")) + goto out_detach; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 2")) + goto out_detach; + + if (event.type == EV_REL && event.code == REL_Y && + event.value == 1) + break; + } + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after IR was decoded")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should still be attached")) + goto out_detach; + + /* + * The lirc chardev can hold more than one attached program at once. + * Load a second, independent instance and check it can be attached + * alongside the first, queried, and then detached on its own + * without disturbing the first program's attachment. + */ + skel2 = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel2, "lirc_mode2__open_and_load (2nd)")) + goto out_detach; + + progfd2 = bpf_program__fd(skel2->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd2, &info, &info_len); + if (!ASSERT_OK(ret, "get second program's info")) + goto out_detach; + prog_id2 = info.id; + + ret = bpf_prog_attach(progfd2, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach second program to lirc device")) + goto out_detach; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second attach")) + goto out_detach2; + if (!ASSERT_EQ(prog_cnt, 2, "two programs should be attached")) + goto out_detach2; + ASSERT_TRUE((prog_ids[0] == prog_id && prog_ids[1] == prog_id2) || + (prog_ids[0] == prog_id2 && prog_ids[1] == prog_id), + "queried ids should be the two attached programs"); + + /* Detach the second program; the first should remain attached. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_OK(ret, "detach second program")) + goto out_detach2; + + /* Detaching an already-detached program should now fail. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach second program again"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second detach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should remain attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "remaining program should be the first one"); + +out_detach: + /* Let's try detaching it now it is actually attached. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_OK(ret, "detach program from lirc device"); + + /* Detaching it again should now fail the same way. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach program from lirc device again"); + goto out; + +out_detach2: + /* Best-effort cleanup of the second program before bailing out. */ + bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + goto out_detach; + +out: + if (inputfd >= 0) + close(inputfd); + if (lircfd >= 0) + close(lircfd); + lirc_mode2__destroy(skel2); + lirc_mode2__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/lirc_mode2.c b/tools/testing/selftests/bpf/progs/lirc_mode2.c new file mode 100644 index 000000000000..98137f3c5c03 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lirc_mode2.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young + +#include +#include +#include + +SEC("lirc_mode2") +int bpf_decoder(unsigned int *sample) +{ + if (LIRC_IS_PULSE(*sample)) { + unsigned int duration = LIRC_VALUE(*sample); + + /* + * Flag bits picked deliberately low: rc-loopback simulates + * a receiver overflow for any pulse over MS_TO_US(50) (see + * loop_tx_ir() in rc-loopback.c), which would silently + * swallow the sample before it ever reaches this decoder. + */ + if (duration & 0x8000) + bpf_rc_keydown(sample, 0x40, duration & 0x3fff, 0); + if (duration & 0x4000) + bpf_rc_pointer_rel(sample, (duration >> 7) & 0x7f, + duration & 0x7f); + } + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c deleted file mode 100644 index cbe4284c032f..000000000000 --- a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young - -#include -#include -#include - -SEC("lirc_mode2") -int bpf_decoder(unsigned int *sample) -{ - if (LIRC_IS_PULSE(*sample)) { - unsigned int duration = LIRC_VALUE(*sample); - - if (duration & 0x1000) - bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); - if (duration & 0x2000) - bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, - duration & 0xff); - } - - return 0; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh deleted file mode 100755 index 5252b91f48a1..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 -ret=$ksft_skip - -msg="skip all tests:" -if [ $UID != 0 ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -GREEN='\033[0;92m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -modprobe rc-loopback - -for i in /sys/class/rc/rc* -do - if grep -q DRV_NAME=rc-loopback $i/uevent - then - LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q) - INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q) - fi -done - -if [ -n "$LIRCDEV" ]; -then - TYPE=lirc_mode2 - ./test_lirc_mode2_user $LIRCDEV $INPUTDEV - ret=$? - if [ $ret -ne 0 ]; then - echo -e ${RED}"FAIL: $TYPE"${NC} - else - echo -e ${GREEN}"PASS: $TYPE"${NC} - fi -fi - -exit $ret diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c deleted file mode 100644 index cd191da20d14..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young - -// A lirc chardev is a device representing a consumer IR (cir) device which -// can receive infrared signals from remote control and/or transmit IR. -// -// IR is sent as a series of pulses and space somewhat like morse code. The -// BPF program can decode this into scancodes so that rc-core can translate -// this into input key codes using the rc keymap. -// -// This test works by sending IR over rc-loopback, so the IR is processed by -// BPF and then decoded into scancodes. The lirc chardev must be the one -// associated with rc-loopback, see the output of ir-keytable(1). -// -// The following CONFIG options must be enabled for the test to succeed: -// CONFIG_RC_CORE=y -// CONFIG_BPF_RAWIR_EVENT=y -// CONFIG_RC_LOOPBACK=y - -// Steps: -// 1. Open the /dev/lircN device for rc-loopback (given on command line) -// 2. Attach bpf_lirc_mode2 program which decodes some IR. -// 3. Send some IR to the same IR device; since it is loopback, this will -// end up in the bpf program -// 4. bpf program should decode IR and report keycode -// 5. We can read keycode from same /dev/lirc device - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bpf_util.h" -#include -#include - -#include "testing_helpers.h" - -int main(int argc, char **argv) -{ - struct bpf_object *obj; - int ret, lircfd, progfd, inputfd; - int testir1 = 0x1ead; - int testir2 = 0x2101; - u32 prog_ids[10], prog_flags[10], prog_cnt; - - if (argc != 3) { - printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]); - return 2; - } - - ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o", - BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); - if (ret) { - printf("Failed to load bpf program\n"); - return 1; - } - - lircfd = open(argv[1], O_RDWR | O_NONBLOCK); - if (lircfd == -1) { - printf("failed to open lirc device %s: %m\n", argv[1]); - return 1; - } - - /* Let's try detach it before it was ever attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret != -ENOENT) { - printf("bpf_prog_detach2 not attached should fail: %m\n"); - return 1; - } - - inputfd = open(argv[2], O_RDONLY | O_NONBLOCK); - if (inputfd == -1) { - printf("failed to open input device %s: %m\n", argv[1]); - return 1; - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 0) { - printf("Expected nothing to be attached\n"); - return 1; - } - - ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); - if (ret) { - printf("Failed to attach bpf to lirc device: %m\n"); - return 1; - } - - /* Write raw IR */ - ret = write(lircfd, &testir1, sizeof(testir1)); - if (ret != sizeof(testir1)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - struct pollfd pfd = { .fd = inputfd, .events = POLLIN }; - struct input_event event; - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_MSC && event.code == MSC_SCAN && - event.value == 0x1ead) { - break; - } - } - - /* Write raw IR */ - ret = write(lircfd, &testir2, sizeof(testir2)); - if (ret != sizeof(testir2)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_REL && event.code == REL_Y && - event.value == 1 ) { - break; - } - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 1) { - printf("Expected one program to be attached\n"); - return 1; - } - - /* Let's try detaching it now it is actually attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret) { - printf("bpf_prog_detach2: returned %m\n"); - return 1; - } - - return 0; -} From 802a30c5db726639df7f93b3a1982ecc64202abe Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 31 Aug 2026 14:32:25 +0800 Subject: [PATCH 073/660] bpf: Fix queue/stack map u32 index overflow The queue/stack map addresses elements[] with the product of a u32 head/tail index and value_size, but the storage itself is allocated in 64-bit arithmetic. When max_entries * value_size reaches or exceeds U32_MAX, the product wraps and push/peek/pop operate on the wrong element, corrupting map data and leaking stale values to user space. max_entries == U32_MAX would also make the u32 capacity counter qs->size (max_entries + 1) wrap to 0 and permanently break the map. The original bound check was removed by commit a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps"), which deleted the bpf_map_charge_init() call and with it the U32_MAX - PAGE_SIZE check that had earlier been moved into bpf_map_charge_init() by c85d69135a91. Oversized queue/stack maps can therefore be created again. Restore the bound in queue_stack_map_alloc_check() with a single comparison that rejects any max_entries/value_size combination whose element storage would reach or exceed U32_MAX bytes, keeping the u32 index multiplication overflow-free and the capacity counter valid. Fixes: a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260831063226.621309-2-chenyuan_fl@163.com --- kernel/bpf/queue_stack_maps.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c32a10..1cee2ba6e332 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; } From 33576bfee5b7f65e3df651ac33f83fdcf1f04233 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 31 Aug 2026 14:32:26 +0800 Subject: [PATCH 074/660] selftests/bpf: Add regression test for queue/stack map size limit Verify that queue/stack maps whose element storage would overflow the u32 head/tail index multiplication are rejected at creation time, and that max_entries == U32_MAX (which would wrap the u32 capacity counter to 0) is rejected as well. Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260831063226.621309-3-chenyuan_fl@163.com --- .../bpf/prog_tests/queue_stack_map.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index 41441325e179..a01a0bcbdee7 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c @@ -2,6 +2,8 @@ #include #include +#define U32_MAX ((u32)UINT_MAX) + enum { QUEUE, STACK, @@ -101,8 +103,35 @@ out: bpf_object__close(obj); } +static void test_queue_stack_map_alloc_check(void) +{ + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_oversize"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1, U32_MAX, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_u32max"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "stack_oversize"); + if (fd >= 0) + close(fd); + + /* A normal-sized map must still be created successfully. */ + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 64, 100, NULL); + ASSERT_GE(fd, 0, "queue_normal"); + if (fd >= 0) + close(fd); +} + void test_queue_stack_map(void) { test_queue_stack_map_by_type(QUEUE); test_queue_stack_map_by_type(STACK); + test_queue_stack_map_alloc_check(); } From 06d6f845c4bda857a6ae864690f35321b878e827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:54 +0200 Subject: [PATCH 075/660] bpftool: Set BPF_F_XDP_DEV_BOUND_ONLY flag non-destructively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When setting the XDP hints ifname, bpftool would set the BPF_F_XDP_DEV_BOUND_ONLY without looking at the existing program flags, overriding any other flag values. This was always a destructive action, but after we change libbpf to carry the frags section flag in prog_flags, this can impact bpftool loading of XDP frags programs. Change the flag setting to be non-destructive by OR'ing it with the existing flags. Fixes: f46392ee3dec ("bpftool: Specify XDP Hints ifname when loading program") Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Reviewed-by: Larysa Zaremba Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-1-4eb6f14968b0@redhat.com --- tools/bpf/bpftool/prog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a9f730d407a9..8c2f9255b36d 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1769,7 +1769,7 @@ offload_dev: } if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) { - bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(pos, bpf_program__flags(pos) | BPF_F_XDP_DEV_BOUND_ONLY); bpf_program__set_ifindex(pos, xdpmeta_ifindex); } else { bpf_program__set_ifindex(pos, offload_ifindex); From d6746a246b362c3c83df170eda6d3c44cc0a4040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:55 +0200 Subject: [PATCH 076/660] selftests/bpf: Set BPF program flags non-destructively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A couple of the BPF selftests would set the program flags without looking at the existing program flags, overriding any other flag values. To avoid issues when we change libbpf to carry section flags in prog_flags, change this to always set the flag value non-destructively by OR'ing it with the existing flags. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-2-4eb6f14968b0@redhat.com --- tools/testing/selftests/bpf/prog_tests/attach_probe.c | 6 ++++-- tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c | 2 +- tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c | 2 +- tools/testing/selftests/bpf/prog_tests/xdp_metadata.c | 4 ++-- tools/testing/selftests/bpf/xdp_hw_metadata.c | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index e8c1a619e330..7dadb90e7b68 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -543,8 +543,10 @@ static void test_kprobe_sleepable(void) return; /* sleepable kprobe test case needs flags set before loading */ - if (!ASSERT_OK(bpf_program__set_flags(skel->progs.handle_kprobe_sleepable, - BPF_F_SLEEPABLE), "kprobe_sleepable_flags")) + if (!ASSERT_OK(bpf_program__set_flags( + skel->progs.handle_kprobe_sleepable, + bpf_program__flags(skel->progs.handle_kprobe_sleepable) | BPF_F_SLEEPABLE), + "kprobe_sleepable_flags")) goto cleanup; if (!ASSERT_OK(test_attach_kprobe_sleepable__load(skel), diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 73f669014b69..652307a1b22e 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, testing_prog_flags()); + bpf_program__set_flags(prog, bpf_program__flags(prog) | testing_prog_flags()); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 2e0ddef77ba5..ed3fd0a88dab 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -362,7 +362,7 @@ static void test_attach_api_fails(void) sl_skel->bss->user_ptr = sl_skel; err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, - BPF_F_SLEEPABLE); + bpf_program__flags(sl_skel->progs.handle_kprobe_multi_sleepable) | BPF_F_SLEEPABLE); if (!ASSERT_OK(err, "sleep_skel_set_flags")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 5c31054ad4a4..047dfdc322a2 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -408,14 +408,14 @@ void test_xdp_metadata(void) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, rx_ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); /* Make sure we can load a dev-bound program that performs * XDP_REDIRECT into a devmap. */ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect"); bpf_program__set_ifindex(new_prog, rx_ifindex); - bpf_program__set_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(new_prog, bpf_program__flags(new_prog) | BPF_F_XDP_DEV_BOUND_ONLY); if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton")) goto out; diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 6db3b5555a22..c5501b3fdf48 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -845,7 +845,7 @@ int main(int argc, char *argv[]) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); printf("load bpf program...\n"); ret = xdp_hw_metadata__load(bpf_obj); From d6b633b585f348053dd27044b1b39972c748c1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:56 +0200 Subject: [PATCH 077/660] libbpf: Move section-defined program flags to prog_flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libbpf section definition modifiers for XDP frags support and sleepable programs stores the flags bits only in the private section definition cookie from object open to load time. This has the unfortunate consequence that API consumers cannot see (or manipulate) the flag between object open and program load. In particular, libxdp has special handling of frags-enabled programs to make them compatible with the dispatcher. This doesn't work on XDP programs that enable frags through the 'xdp.frags' section definition because the flag is not visible through bpf_program__flags()[0]. Fix this by changing how libbpf loads the program flags from section definitions: instead of using the private section definition cookie, add a setup function to the default section definitions that stores the flags for sleepable and XDP frags programs in the prog_flags field of struct bpf_program. Exposing the flags this way means that any use of bpf_program__set_flags() will override the flags unless the caller takes care of updating flags in a non-destructive way. This is unavoidable with the set-only API, and any user setting flags unconditionally is already broken in the sense that they will also override any other current and future flags. In addition, prog_flags survives program type changes through bpf_program__set_type(). It is the responsibility of the caller to ensure the flags are cleared if they are incompatible with the new program type. [0] https://github.com/xdp-project/xdp-tools/issues/587 Fixes: 082c4bfba4f7 ("libbpf: Add SEC name for xdp frags programs") Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-3-4eb6f14968b0@redhat.com --- tools/lib/bpf/libbpf.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index f09cbfd8e729..c036e8a91ed8 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -7879,6 +7879,19 @@ static int tracing_multi_mod_fd(struct bpf_program *prog, int *btf_obj_fd) return 0; } +static int libbpf_setup_prog_flags(struct bpf_program *prog, long cookie) +{ + enum sec_def_flags def = cookie; + + if (def & SEC_SLEEPABLE) + prog->prog_flags |= BPF_F_SLEEPABLE; + + if (def & SEC_XDP_FRAGS) + prog->prog_flags |= BPF_F_XDP_HAS_FRAGS; + + return 0; +} + /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ static int libbpf_prepare_prog_load(struct bpf_program *prog, struct bpf_prog_load_opts *opts, long cookie) @@ -7889,12 +7902,6 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) opts->expected_attach_type = 0; - if (def & SEC_SLEEPABLE) - opts->prog_flags |= BPF_F_SLEEPABLE; - - if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) - opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; - /* special check for usdt to use uprobe_multi link */ if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type @@ -10099,6 +10106,7 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts .prog_type = BPF_PROG_TYPE_##ptype, \ .expected_attach_type = atype, \ .cookie = (long)(flags), \ + .prog_setup_fn = libbpf_setup_prog_flags, \ .prog_prepare_load_fn = libbpf_prepare_prog_load, \ __VA_ARGS__ \ } From 99c71666dcb11b92bacedb7ddc5bf172aedc07c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:57 +0200 Subject: [PATCH 078/660] selftests/bpf: Check for flag presence in bpf_program__flags() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a check that the BPF_F_XDP_HAS_FRAGS and BPF_F_SLEEPABLE flags show up in bpf_program__flags() when opening a BPF program with the flag definitions in their section definitions. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-4-4eb6f14968b0@redhat.com --- tools/testing/selftests/bpf/prog_tests/kernel_flag.c | 3 +++ tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c | 3 +++ tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c index 97b00c7efe94..25eb59f460ab 100644 --- a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c +++ b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c @@ -16,6 +16,9 @@ void test_kernel_flag(void) if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel")) return; + ASSERT_EQ(bpf_program__flags(lsm_skel->progs.bpf) & BPF_F_SLEEPABLE, + BPF_F_SLEEPABLE, "sleepable in program flags"); + lsm_skel->bss->monitored_tid = sys_gettid(); ret = test_kernel_flag__attach(lsm_skel); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c index fce203640f8c..a894b1ab46f4 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c @@ -18,6 +18,9 @@ static void test_xdp_update_frags(void) return; prog = bpf_object__next_program(obj, NULL); + ASSERT_EQ(bpf_program__flags(prog) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + if (bpf_object__load(obj)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c index a8ab05216c38..dff6b3e7266e 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c @@ -146,6 +146,11 @@ static void test_xdp_with_devmap_frags_helpers(void) if (!ASSERT_OK_PTR(skel, "test_xdp_with_devmap_helpers__open_and_load")) return; + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm_frags) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm) & BPF_F_XDP_HAS_FRAGS, + 0, "frags not in program flags"); + dm_fd_frags = bpf_program__fd(skel->progs.xdp_dummy_dm_frags); map_fd = bpf_map__fd(skel->maps.dm_ports); err = bpf_prog_get_info_by_fd(dm_fd_frags, &info, &len); From 7bf591f26545f5776a4e42d08c06fd94469766dd Mon Sep 17 00:00:00 2001 From: Sudip Mukherjee Date: Mon, 31 Aug 2026 14:45:37 +0100 Subject: [PATCH 079/660] libbpf: Fix debian kernel version information parsing Currently get_debian_kernel_version() expects the version to be in the form of x.y.z but the Debian kernels uploaded to experimental are not always LTS kernels. Like the current Debian kernel in experimental is 7.2~rc7-1~exp1 and uname -v reported "Debian 7.2~rc7-1~exp1". Signed-off-by: Sudip Mukherjee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260831134537.1227914-1-sudipm.mukherjee@gmail.com --- tools/lib/bpf/libbpf_probes.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index e40819465ddc..5acf0f84a40a 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -62,7 +62,7 @@ static __u32 get_ubuntu_kernel_version(void) */ static __u32 get_debian_kernel_version(struct utsname *info) { - __u32 major, minor, patch; + __u32 major, minor, patch = 0; char *p; p = strstr(info->version, "Debian "); @@ -71,10 +71,13 @@ static __u32 get_debian_kernel_version(struct utsname *info) return 0; } - if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) != 3) - return 0; + if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) == 3) + return KERNEL_VERSION(major, minor, patch); - return KERNEL_VERSION(major, minor, patch); + if (sscanf(p, "Debian %u.%u", &major, &minor) == 2) + return KERNEL_VERSION(major, minor, patch); + + return 0; } __u32 get_kernel_version(void) From d02bfbc49c8ef978eac5f39a2de67281b8e00edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:30 +0200 Subject: [PATCH 080/660] bpf: mark instructions accessing program stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare to emit KASAN checks in JITed programs, JIT compilers need to be aware about whether some load/store instructions are targeting the bpf program stack, as those should not be monitored (we already have guard pages for that, and it is difficult anyway to correctly monitor any kind of data passed on stack). To support this need, make the BPF verifier mark the instructions depending on whether they could access or not memory other than stack. As different states in the verifier could lead to different memory types for the same access, just marking an instruction as accessing stack only is not enough (it could be some other memory type in another verifier state), so the algorithm rather sets by default any load/store instruction as stack only, and if _any_ state leads to any memory access type other than PTR_TO_STACK, it overrides this setting. It also takes care about shifting back the instruction marking in adjust_insn_aux_data if the verifier patches instructions. However, if the verifier generates new BPF_ST/BPF_STX/BPF_LDX while patching some instructions, those new ones are systematically marked as non-stack-accessing: this may over-instrument a few memory accessing instructions, but it allows making sure that we will not miss accidentally any. Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-1-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 16 ++++++++++++++++ kernel/bpf/fixups.c | 33 ++++++++++++++++++++++++++++++--- kernel/bpf/verifier.c | 10 ++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae9f606539f4..1a3c44ab06a1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -706,6 +706,7 @@ struct bpf_insn_aux_data { */ u32 calls_callback:1; u32 indirect_target:1; /* if it is an indirect jump target */ + u32 non_stack_access:1; /* instruction can access non-stack memory */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -1671,6 +1672,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..73fb3ffc18e3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -213,7 +213,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -227,8 +228,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -239,8 +247,25 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is @@ -306,6 +331,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + struct bpf_insn original_insn; if (len > 1) { new_data = vrealloc(env->insn_aux_data, @@ -318,6 +344,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -326,7 +353,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f585ceb2cd5..5b51e7ee1a3f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3273,6 +3273,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + #define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) @@ -6417,6 +6422,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6655,6 +6661,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } From 90769ad233cac2ade07f8d2f91214e584842158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:31 +0200 Subject: [PATCH 081/660] bpf: add BPF_JIT_KASAN for KASAN instrumentation of JITed programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new Kconfig option CONFIG_BPF_JIT_KASAN that automatically enables generic KASAN (Kernel Address SANitizer) memory access checks for JIT-compiled BPF programs as well, when both KASAN (and more specifically, generic KASAN with KASAN_VMALLOC) and JIT compiler are enabled. This new Kconfig is not a user selectable one: it is automatically enabled if KASAN is enabled on a compatible platform. When enabled, the JIT compiler will emit shadow memory checks before memory loads and stores to detect use-after-free or out-of-bounds accesses at runtime. The option is gated behind HAVE_EBPF_JIT_KASAN, as it needs proper arch-specific implementation. Acked-by: Andrey Konovalov Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-2-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/Kconfig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f0..d7d25477ef48 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -101,4 +105,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" From b3b45e9d72dfdc3534b04747978c891b0b30fc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:32 +0200 Subject: [PATCH 082/660] bpf, x86: refactor BPF_ST management in do_jit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare for KASAN checks insertion before every memory-related load or store, group all BPF_ST instructions that indeed access memory in a single helper to allow instrumenting those in one call, rather than having to instrument all cases individually. Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-3-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 73 +++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 48429fae0641..f907be3988b2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1315,6 +1315,46 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int *pprog = prog; } +/* ST: *(u8*)(dst_reg + off) = imm */ +static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + s32 insn_off) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + if (is_imm8(insn_off)) + EMIT2(add_1reg(0x40, dst_reg), insn_off); + else + EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -2250,49 +2290,28 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + imm32); break; } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; + insn_off = insn->off; if (dst_reg == BPF_REG_PARAMS) { /* * Args 7+: reverse BPF negative offsets to * x86 positive rsp offsets. * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; dst_reg = BPF_REG_FP; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); - - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, insn_off); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ From 92b73b708b1764b3f3a02652c1e8dd3965b6d1ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:33 +0200 Subject: [PATCH 083/660] bpf, x86: emit KASAN checks in x86 JITed programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Insert KASAN shadow memory checks before memory load and store operations in JIT-compiled BPF programs. This helps detect memory safety bugs such as use-after-free and out-of-bounds accesses at runtime. The main instructions being targeted are BPF_ST, BPF_STX and BPF_LDX, but not all of them are being instrumented: - if the load/store instruction is in fact accessing the program stack, emit_kasan_check silently skips the instrumentation, as we can already benefit from guard pages to monitor stack accesses. - if the load/store instruction is a BPF_PROBE_MEM or a BPF_PROBE_ATOMIC instruction, we do not instrument it, as the passed address can fault (hence the custom fault management with BPF_PROBE_XXX instructions), and so the corresponding kasan check could fault as well. To support those new instructions insertion, create the emit_kasan_check() helper that emits KASAN shadow memory checks before memory accesses in JIT-compiled BPF programs. The implementation relies on the existing __asan_{load,store}X functions from KASAN subsystem. The helper: - saves registers. This includes caller-saved registers, but also temporary registers, as those were possibly used by the affected program. - computes the accessed address and stores it in %rdi - calls the relevant function, depending on the instruction being a load or a store, and the size of the access. - restores registers The special care needed when inserting this instrumentation comes at the cost of a non negligeable increase in JITed code size. For example, a bare mov 0x0(%si),rbx # Load in rbx content at address stored in rsi becomes push %rax push %rcx push %rdx push %rsi push %rdi push %r8 push %r9 push %r10 push %r11 mov %rsi,%rdi call 0xffffffff81da0a60 <__asan_load8> pop %r11 pop %r10 pop %r9 pop %r8 pop %rdi pop %rsi pop %rdx pop %rcx pop %rax mov 0x0(%rsi),rbx Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-4-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 197 ++++++++++++++++++++++++++++++++---- 1 file changed, 179 insertions(+), 18 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index f907be3988b2..b8c2e935689e 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -21,6 +21,17 @@ #include #include +#if IS_ENABLED(CONFIG_BPF_JIT_KASAN) +void __asan_load1(void *p); +void __asan_store1(void *p); +void __asan_load2(void *p); +void __asan_store2(void *p); +void __asan_load4(void *p); +void __asan_store4(void *p); +void __asan_load8(void *p); +void __asan_store8(void *p); +#endif + static bool all_callee_regs_used[4] = {true, true, true, true}; static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) @@ -1110,6 +1121,93 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, + u32 addr_reg, s32 off, u32 bpf_size, u8 *ip, + bool is_write) +{ +#ifdef CONFIG_BPF_JIT_KASAN + u8 *prog = *pprog; + void *kasan_func; + + if (!env) + return 0; + + if (addr_reg == BPF_REG_FP || addr_reg == BPF_REG_PARAMS) + return 0; + + /* Derive KASAN check function from access type and size */ + switch (bpf_size) { + case BPF_B: + kasan_func = is_write ? __asan_store1 : __asan_load1; + break; + case BPF_H: + kasan_func = is_write ? __asan_store2 : __asan_load2; + break; + case BPF_W: + kasan_func = is_write ? __asan_store4 : __asan_load4; + break; + case BPF_DW: + kasan_func = is_write ? __asan_store8 : __asan_load8; + break; + default: + return -EINVAL; + } + + /* Save rax */ + EMIT1(0x50); + /* Save rcx */ + EMIT1(0x51); + /* Save rdx */ + EMIT1(0x52); + /* Save rsi */ + EMIT1(0x56); + /* Save rdi */ + EMIT1(0x57); + /* Save r8 */ + EMIT2(0x41, 0x50); + /* Save r9 */ + EMIT2(0x41, 0x51); + /* Save r10 */ + EMIT2(0x41, 0x52); + /* Save r11 */ + EMIT2(0x41, 0x53); + /* mov rdi, addr_reg */ + EMIT_mov(BPF_REG_1, addr_reg); + + /* add rdi, off (if offset is non-zero) */ + if (off) { + if (is_imm8(off)) { + /* add rdi, imm8 */ + EMIT4(0x48, 0x83, 0xC7, (u8)off); + } else { + /* add rdi, imm32 */ + EMIT3_off32(0x48, 0x81, 0xC7, off); + } + } + + /* Adjust ip to account for the instrumentation generated so far */ + ip += (prog - *pprog); + /* We emit a call, so update call depth counting */ + ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip); + /* call kasan_func */ + if (emit_call(&prog, kasan_func, ip)) + return -ERANGE; + + EMIT2(0x41, 0x5B); + EMIT2(0x41, 0x5A); + EMIT2(0x41, 0x59); + EMIT2(0x41, 0x58); + EMIT1(0x5F); + EMIT1(0x5E); + EMIT1(0x5A); + EMIT1(0x59); + EMIT1(0x58); + + *pprog = prog; +#endif /* CONFIG_BPF_JIT_KASAN */ + return 0; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1463,17 +1561,35 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) +static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog, + struct bpf_insn *insn, u8 *ip, u32 dst_reg, + u32 src_reg, bool accesses_stack_only) { + u32 atomic_op = insn->imm; + int err; + switch (atomic_op) { case BPF_LOAD_ACQ: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, src_reg, insn->off, + BPF_SIZE(insn->code), ip, false); + if (err) + return err; + } /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); + emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; case BPF_STORE_REL: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, dst_reg, insn->off, + BPF_SIZE(insn->code), ip, true); + if (err) + return err; + } /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); + emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; default: pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", @@ -1894,10 +2010,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * const s32 imm32 = insn->imm; u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; + bool accesses_stack_only; u8 b2 = 0, b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; + int insn_idx; u8 jmp_cond; u8 *func; int nops; @@ -1914,6 +2032,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_ENDBR(); ip = image + addrs[i - 1] + (prog - temp); + insn_idx = i - 1 + bpf_prog->aux->subprog_start; + accesses_stack_only = + env ? !env->insn_aux_data[insn_idx].non_stack_access : + false; switch (insn->code) { /* ALU */ @@ -2311,6 +2433,15 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } + emit_st(&prog, insn, dst_reg, insn_off); break; @@ -2329,6 +2460,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); break; @@ -2510,6 +2649,13 @@ populate_extable: /* populate jmp_offset for JAE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; + } else if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, src_reg, + insn_off, + BPF_SIZE(insn->code), ip, + false); + if (err) + return err; } if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX || BPF_MODE(insn->code) == BPF_MEMSX) @@ -2571,28 +2717,42 @@ populate_extable: } fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; - u32 real_src_reg = src_reg; - u32 real_dst_reg = dst_reg; - u8 *branch_target; - + case BPF_STX | BPF_ATOMIC | BPF_DW: { + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + u32 real_src_reg = src_reg; + u32 real_dst_reg = dst_reg; + u8 *old_prog; + bool is_atomic_fetch = + (insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH)); + if (is_atomic_fetch) { /* * Can't be implemented with a single x86 insn. * Need to do a CMPXCHG loop. */ /* Will need RAX as a CMPXCHG operand so save R0 */ + old_prog = prog; emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); if (src_reg == BPF_REG_0) real_src_reg = BPF_REG_AX; if (dst_reg == BPF_REG_0) real_dst_reg = BPF_REG_AX; - - branch_target = prog; + ip += (prog - old_prog); + } + if (!bpf_atomic_is_load_store(insn)) { + if (!accesses_stack_only) { + err = emit_kasan_check( + env, &prog, real_dst_reg, + insn->off, BPF_SIZE(insn->code), + ip, true); + if (err) + return err; + } + } + if (is_atomic_fetch) { + u8 *branch_target = prog; /* Load old value */ emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, real_dst_reg, insn->off); @@ -2624,15 +2784,16 @@ populate_extable: } if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + err = emit_atomic_ld_st(env, &prog, insn, ip, + dst_reg, src_reg, + accesses_stack_only); else err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg, insn->off, BPF_SIZE(insn->code)); if (err) return err; break; - + } case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: if (!bpf_atomic_is_load_store(insn)) { From cfc821c029fd66237b9a1f66362d93d4a5cab48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:34 +0200 Subject: [PATCH 084/660] bpf, x86: enable KASAN for JITed programs on x86 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark x86 as supporting KASAN checks in JITed programs so that the corresponding JIT compiler inserts checks on the translated instructions. Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-5-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 48ccc3e6059d..745890d91e99 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -234,6 +234,7 @@ config X86 select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT + select HAVE_EBPF_JIT_KASAN if X86_64 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD From 040ef1f8b02e57b78081581d490ceec04ef5f7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:35 +0200 Subject: [PATCH 085/660] selftests/bpf: make cmdline_contains stricter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cmdline_contains is used by BPF selftests to check the presence of specific kernel commandline parameters, but it currently suffers from two issues: - the read commandline isn't NULL terminated right after the read data but only at the end of the buffer, leaving uninitialized bytes that are then possibly tokenized - the comparison of found tokens is done based on the size of found token. This could lead to too-short-but-matching tokens to wrongly match the search pattern. Enforce stricter checks in cmdline_contains to avoid accidental matches. Fixes: 399f6185a1c0 ("selftests/bpf: Fix selftests broken by mitigations=off") Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-6-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/unpriv_helpers.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index f997d7ec8fd0..9dadcacaef0c 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -72,8 +72,8 @@ static int config_contains(const char *pat) static bool cmdline_contains(const char *pat) { + int fd, cnt, ret = false; char cmdline[4096], *c; - int fd, ret = false; fd = open("/proc/cmdline", O_RDONLY); if (fd < 0) { @@ -81,14 +81,15 @@ static bool cmdline_contains(const char *pat) return false; } - if (read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + cnt = read(fd, cmdline, sizeof(cmdline) - 1); + if (cnt < 0) { perror("read /proc/cmdline"); goto out; } - cmdline[sizeof(cmdline) - 1] = '\0'; + cmdline[cnt] = '\0'; for (c = strtok(cmdline, " \n"); c; c = strtok(NULL, " \n")) { - if (strncmp(c, pat, strlen(c))) + if (strcmp(c, pat)) continue; ret = true; break; From 9123a4d4c4f3321efbb40569b81985d9654abe25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:36 +0200 Subject: [PATCH 086/660] selftests/bpf: add helpers for KASAN in JIT testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two simple helpers to allow checking whether KASAN for eBPF tests should be executed: - one helper to check if BPF_JIT_KASAN is enabled in kernel configuration - one helper to check if the kernel is running with kasan_multi_shot (otherwise only the first test will be able to trigger a report) Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-7-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/unpriv_helpers.c | 10 ++++++++++ tools/testing/selftests/bpf/unpriv_helpers.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index 9dadcacaef0c..2c8c5edb8751 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -143,3 +143,13 @@ bool get_unpriv_disabled(void) } return mitigations_off; } + +bool get_kasan_jit_enabled(void) +{ + return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; +} + +bool get_kasan_multi_shot_enabled(void) +{ + return cmdline_contains("kasan_multi_shot"); +} diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index 151f67329665..a7ceb51577cd 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h @@ -5,3 +5,5 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_kasan_jit_enabled(void); +bool get_kasan_multi_shot_enabled(void); From 2cbbb035977a79b78952404d3e2c3c6dfe318259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:37 +0200 Subject: [PATCH 087/660] selftests/bpf: add tests to validate KASAN on JIT programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a basic KASAN test runner that loads and test-run programs that can trigger memory management bugs. The test captures kernel logs and ensure that the expected KASAN splat is emitted by searching for the corresponding first lines in the report, hence validated that the needed instrumentation has been inserted by the JIT compiler before the relevant memory accesses. To allow each test to trigger the expected report, the kernel must run with the kasan_multi_shot configuration. The runner covers different cases and settings: in the nominal case, it validates kasan reports on basic instructions (on all supported accesses sizes) but also when report _should not_ be emitted (eg: for accesses on program stack). The runner also comes with a few specialized tests that are then not executed for all sizes/locations: - specific atomic ops - test for instructions involving different verifier states, with some states flagging memory as stack, and other states as non-stack memory - tests that validate the stack marking shifting when a patch is emitted by the verifier (zext/rnd_hi32, constant blindind). Most of those tests are able to trigger kasan reports by altering the shadow memory (triggering faulty accesses is otherwise complex, because of the verifier). A few tests trigger actual faulty accesses (eg out-of-bound accesses) A few of those tests depends on cpuv4 (load_acquire/store_release, st, st_blinded), and so are executed only with test_progs-cpuv4 # ./test_progs -a kasan #175/1 kasan/st_1_not_on_stack:SKIP #175/2 kasan/st_1_on_stack:SKIP #175/3 kasan/st_2_not_on_stack:SKIP #175/4 kasan/st_2_on_stack:SKIP #175/5 kasan/st_4_not_on_stack:SKIP #175/6 kasan/st_4_on_stack:SKIP #175/7 kasan/st_8_not_on_stack:SKIP #175/8 kasan/st_8_on_stack:SKIP #175/9 kasan/stx_1_not_on_stack:OK #175/10 kasan/stx_1_on_stack:OK #175/11 kasan/stx_2_not_on_stack:OK #175/12 kasan/stx_2_on_stack:OK #175/13 kasan/stx_4_not_on_stack:OK #175/14 kasan/stx_4_on_stack:OK #175/15 kasan/stx_8_not_on_stack:OK #175/16 kasan/stx_8_on_stack:OK #175/17 kasan/ldx_1_not_on_stack:OK #175/18 kasan/ldx_1_on_stack:OK #175/19 kasan/ldx_2_not_on_stack:OK #175/20 kasan/ldx_2_on_stack:OK #175/21 kasan/ldx_4_not_on_stack:OK #175/22 kasan/ldx_4_on_stack:OK #175/23 kasan/ldx_8_not_on_stack:OK #175/24 kasan/ldx_8_on_stack:OK #175/25 kasan/simple_atomic_4_not_on_stack:OK #175/26 kasan/simple_atomic_4_on_stack:OK #175/27 kasan/simple_atomic_8_not_on_stack:OK #175/28 kasan/simple_atomic_8_on_stack:OK #175/29 kasan/simple_atomic_fetch_not_on_stack:OK #175/30 kasan/simple_atomic_fetch_on_stack:OK #175/31 kasan/load_acquire_1_not_on_stack:SKIP #175/32 kasan/load_acquire_1_on_stack:SKIP #175/33 kasan/load_acquire_2_not_on_stack:SKIP #175/34 kasan/load_acquire_2_on_stack:SKIP #175/35 kasan/load_acquire_4_not_on_stack:SKIP #175/36 kasan/load_acquire_4_on_stack:SKIP #175/37 kasan/load_acquire_8_not_on_stack:SKIP #175/38 kasan/load_acquire_8_on_stack:SKIP #175/39 kasan/store_release_1_not_on_stack:SKIP #175/40 kasan/store_release_1_on_stack:SKIP #175/41 kasan/store_release_2_not_on_stack:SKIP #175/42 kasan/store_release_2_on_stack:SKIP #175/43 kasan/store_release_4_not_on_stack:SKIP #175/44 kasan/store_release_4_on_stack:SKIP #175/45 kasan/store_release_8_not_on_stack:SKIP #175/46 kasan/store_release_8_on_stack:SKIP #175/47 kasan/ldx_patched_not_on_stack:OK #175/48 kasan/ldx_patched_on_stack:OK #175/49 kasan/verifier_paths_stack_and_non_stack:OK #175/50 kasan/ldx_oob_1_not_on_stack:OK #175/51 kasan/ldx_oob_2_not_on_stack:OK #175/52 kasan/ldx_oob_4_not_on_stack:OK #175/53 kasan/ldx_oob_8_not_on_stack:OK #175/54 kasan/ldx_self_alias_on_stack:OK #175/55 kasan/st_blinded:SKIP #175 kasan:OK (SKIP: 25/55) Summary: 1/30 PASSED, 25 SKIPPED, 0/0 FAILED Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-8-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- .../testing/selftests/bpf/prog_tests/kasan.c | 479 +++++++++++++++++ tools/testing/selftests/bpf/progs/kasan.c | 502 ++++++++++++++++++ .../selftests/bpf/progs/kasan_harden.c | 52 ++ .../selftests/bpf/test_kmods/bpf_testmod.c | 55 ++ 4 files changed, 1088 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/kasan.c create mode 100644 tools/testing/selftests/bpf/progs/kasan.c create mode 100644 tools/testing/selftests/bpf/progs/kasan_harden.c diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c new file mode 100644 index 000000000000..7cd4d1208c3b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +/* + * Tests validating that KASAN reports are properly instrumented and + * generated on a wide variety of instructions. The running kernel needs + * kasan_multi_shot to run multiple kasan-generating subtests at once + */ +#include +#include +#include +#include +#include +#include +#include +#include "sysctl_helpers.h" +#include "kasan.skel.h" +#include "kasan_harden.skel.h" + +#define SUBTEST_NAME_MAX_LEN 128 +#define PROG_NAME_MAX_LEN 128 + +#define MAX_LOG_SIZE (8 * 1024) +#define READ_CHUNK_SIZE 256 + +#define KASAN_PATTERN_SLAB_UAF "BUG: KASAN: slab-use-after-free " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_SLAB_OOB "BUG: KASAN: slab-out-of-bounds " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_REPORT "%s of size %d at addr" + +static char klog_buffer[MAX_LOG_SIZE]; +static char record[MAX_LOG_SIZE]; + +struct test_spec { + char *prog_type; + bool is_write; + bool only_32_or_64; + bool needs_load_acq_store_rel; + bool needs_st; + bool skip_multi_size_testing; + bool skip_on_stack_testing; + int run_size; + bool expect_no_report; + bool rnd_hi32; + bool is_oob; +}; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct test_ctx { + __u8 prog_tag[BPF_TAG_SIZE]; + struct bpf_object *obj; + int *access_size; + bool skip_load_acq_store_rel; + bool skip_st_tests; + struct bpf_program *prog; + char prog_name[SUBTEST_NAME_MAX_LEN]; + int klog_fd; +}; + +static int open_kernel_logs(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + + return fd; +} + +static void skip_kernel_logs(int fd) +{ + lseek(fd, 0, SEEK_END); +} + +static int read_kernel_logs(int fd, char *buf, size_t max_len) +{ + size_t total = 0; + ssize_t n; + + buf[0] = '\0'; + while (1) { + char *msg, *eol; + size_t len; + + n = read(fd, record, sizeof(record) - 1); + if (n == 0) + break; + + if (n < 0) { + if (errno == EAGAIN) + break; + return n; + } + record[n] = '\0'; + + /* + * Each kmsg record starts with some metadata, separated + * from the actual content by a semi-colon + */ + msg = strchr(record, ';'); + if (!msg) + continue; + msg++; + eol = strchr(msg, '\n'); + if (eol) + *eol = '\0'; + + len = strlen(msg); + if (total + len + 2 > max_len) + break; + memcpy(buf + total, msg, len); + total += len; + buf[total++] = '\n'; + buf[total] = '\0'; + } + + return total; +} + +static int check_kasan_report_in_kernel_logs(char *buf, struct test_ctx *ctx, + bool is_write, int size, + bool is_oob) +{ + char access_log[READ_CHUNK_SIZE]; + const char *pattern; + char *kasan_report_start; + int nsize; + + pattern = is_oob ? KASAN_PATTERN_SLAB_OOB : KASAN_PATTERN_SLAB_UAF; + nsize = snprintf(access_log, READ_CHUNK_SIZE, pattern, + ctx->prog_tag[0], ctx->prog_tag[1], ctx->prog_tag[2], + ctx->prog_tag[3], ctx->prog_tag[4], ctx->prog_tag[5], + ctx->prog_tag[6], ctx->prog_tag[7], ctx->prog_name); + if (!ASSERT_GE(nsize, 0, "format kasan access header line")) + return nsize; + /* + * Searched kasan report is valid if + * - it contains the expected kasan pattern + * - the description of the faulty access is found somewhere + * after the header (not necessarily on the very next line, + * because other kernel messages may interleave) + * - faulty access properties match the tested type and size + */ + kasan_report_start = strstr(buf, access_log); + + if (!kasan_report_start) + return 1; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_REPORT, + is_write ? "Write" : "Read", size); + if (!ASSERT_GE(nsize, 0, "format kasan access report line")) + return nsize; + + if (!strstr(kasan_report_start, access_log)) + return 1; + + return 0; +} + +static void exec_subtest(struct test_ctx *ctx, struct test_spec *test, + int access_size, bool on_stack) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_info info; + uint8_t buf[ETH_HLEN] = {0}; + int ret, prog_fd; + __u32 info_len; + + ctx->prog = bpf_object__find_program_by_name(ctx->obj, + ctx->prog_name); + if (!ASSERT_OK_PTR(ctx->prog, "find test prog")) + return; + + info_len = sizeof(info); + memset(&info, 0, info_len); + prog_fd = bpf_program__fd(ctx->prog); + if (!ASSERT_OK_FD(prog_fd, "get prog fd")) + return; + ret = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (!ASSERT_OK(ret, "fetch loaded program info")) + return; + memcpy(ctx->prog_tag, info.tag, BPF_TAG_SIZE); + + skip_kernel_logs(ctx->klog_fd); + + topts.sz = sizeof(struct bpf_test_run_opts); + topts.data_size_in = ETH_HLEN; + topts.data_in = buf; + if (ctx->access_size) + *ctx->access_size = access_size; + ret = bpf_prog_test_run_opts(bpf_program__fd(ctx->prog), + &topts); + if (!ASSERT_OK(ret, "run prog")) + return; + + ret = read_kernel_logs(ctx->klog_fd, klog_buffer, MAX_LOG_SIZE); + if (!ASSERT_GE(ret, 0, "read kernel logs")) + return; + + ret = check_kasan_report_in_kernel_logs(klog_buffer, ctx, + test->is_write, access_size, + test->is_oob); + if (on_stack || test->expect_no_report) + ASSERT_NEQ(ret, 0, "no report should be generated"); + else + ASSERT_OK(ret, "report should be generated"); +} + +static void run_subtest_with_size_and_location(struct test_ctx *ctx, + struct test_spec *test, + int access_size, + bool on_stack) +{ + char subtest_name[SUBTEST_NAME_MAX_LEN]; + + if (test->skip_multi_size_testing) { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s%s", + test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + } else { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s_%d_%s", + test->prog_type, access_size, + on_stack ? "on_stack" : "not_on_stack"); + } + + snprintf(ctx->prog_name, PROG_NAME_MAX_LEN, "%s%s", test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + + if (!test__start_subtest(subtest_name)) + return; + + if (test->needs_load_acq_store_rel && ctx->skip_load_acq_store_rel) { + test__skip(); + return; + } + + if (test->needs_st && ctx->skip_st_tests) { + test__skip(); + return; + } + + exec_subtest(ctx, test, access_size, on_stack); +} + +static void run_subtest_with_size(struct test_ctx *ctx, struct test_spec *test, + int size) +{ + run_subtest_with_size_and_location(ctx, test, size, false); + if (!test->skip_on_stack_testing) + run_subtest_with_size_and_location(ctx, test, size, true); +} + +static void run_subtest(struct test_ctx *ctx, struct test_spec *test) +{ + if (test->skip_multi_size_testing) { + run_subtest_with_size(ctx, test, test->run_size); + return; + } + + if (!test->only_32_or_64) { + run_subtest_with_size(ctx, test, 1); + run_subtest_with_size(ctx, test, 2); + } + run_subtest_with_size(ctx, test, 4); + run_subtest_with_size(ctx, test, 8); +} + +static void run_blinding_subtest(void) +{ + struct test_spec blinding_spec = { + .prog_type = "st_blinded", + .is_write = true, + }; + char bpf_jit_harden_orig[2]; + struct kasan_harden *skel; + struct test_ctx *ctx; + + if (!test__start_subtest("st_blinded")) + return; + + ctx = calloc(1, sizeof(*ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc blinding ctx")) + return; + ctx->klog_fd = -1; + + if (sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", + bpf_jit_harden_orig, "2")) + goto free_ctx; + + skel = kasan_harden__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load blinded prog")) + goto restore; + + if (skel->data->skip_st_tests) { + test__skip(); + goto destroy; + } + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + ctx->obj = skel->obj; + strncpy(ctx->prog_name, "st_blinded", PROG_NAME_MAX_LEN); + + exec_subtest(ctx, &blinding_spec, 1, false); + +destroy: + close(ctx->klog_fd); + kasan_harden__destroy(skel); +restore: + sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", NULL, + bpf_jit_harden_orig); +free_ctx: + free(ctx); +} + +static struct test_spec tests[] = { + { + .prog_type = "st", + .is_write = true, + .needs_st = true + }, + { + .prog_type = "stx", + .is_write = true + }, + { + .prog_type = "ldx", + .is_write = false + }, + { + .prog_type = "simple_atomic", + .is_write = true, + .only_32_or_64 = true + }, + { + .prog_type = "simple_atomic_fetch", + .is_write = true, + .skip_multi_size_testing = true, + .run_size = 8, + }, + { + .prog_type = "load_acquire", + .is_write = false, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "store_release", + .is_write = true, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "ldx_patched", + .is_write = false, + .skip_multi_size_testing = true, + .run_size = 4, + .rnd_hi32 = true + }, + { + .prog_type = "verifier_paths_stack_and_non_stack", + .is_write = true, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 1 + }, + { + .prog_type = "ldx_oob", + .is_write = false, + .skip_on_stack_testing = true, + .is_oob = true + }, + { + .prog_type = "ldx_self_alias_on_stack", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 8, + .expect_no_report = true + } +}; + +void serial_test_kasan(void) +{ + struct kasan_write_val val; + struct test_spec *test; + struct test_ctx *ctx; + struct kasan *skel; + __u32 key = 0; + int i, ret; + + ctx = calloc(1, sizeof(struct test_ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc test ctx")) + return; + + if (!is_jit_enabled() || !get_kasan_jit_enabled() || + !get_kasan_multi_shot_enabled()) { + test__skip(); + goto end; + } + + skel = kasan__open(); + if (!ASSERT_OK_PTR(skel, "open prog")) + goto end; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + char prog_name[SUBTEST_NAME_MAX_LEN]; + struct bpf_program *prog; + + if (!tests[i].rnd_hi32) + continue; + + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 on_stack prog")) + goto destroy; + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "not_on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 not_on_stack prog")) + goto destroy; + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + } + + if (!ASSERT_OK(kasan__load(skel), "load prog")) + goto destroy; + + ctx->obj = skel->obj; + ctx->access_size = &skel->bss->access_size; + ctx->skip_load_acq_store_rel = skel->data->skip_load_acq_store_rel_tests; + ctx->skip_st_tests = skel->data->skip_st_tests; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + /* Fill map with recognizable values */ + ret = bpf_map__lookup_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "get map")) + goto close; + val.data_1 = 0xAA; + val.data_2 = 0xBBBB; + val.data_4 = 0xCCCCCCCC; + val.data_8 = 0xDDDDDDDDDDDDDDDD; + ret = bpf_map__update_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "set map")) + goto close; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test = &tests[i]; + run_subtest(ctx, test); + } + + /* + * Blinding subtest is handled differently as it needs the + * corresponding program to be loaded with bpf_jit_harden raised + */ + run_blinding_subtest(); + +close: + close(ctx->klog_fd); +destroy: + kasan__destroy(skel); +end: + free(ctx); +} diff --git a/tools/testing/selftests/bpf/progs/kasan.c b/tools/testing/selftests/bpf/progs/kasan.c new file mode 100644 index 000000000000..fe8e0dd228a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include +#include +#include "bpf_misc.h" + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + +extern struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) __ksym; +extern void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) __ksym; + +int access_size; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val.data_1 = 0xAA; + break; + case 2: + val.data_2 = 0xAA; + break; + case 4: + val.data_4 = 0xAA; + break; + case 8: + val.data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int st_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = 0xAA; + break; + case 2: + val->data_2 = 0xAA; + break; + case 4: + val->data_4 = 0xAA; + break; + case 8: + val->data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int stx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + /* + * Unlike the st() programs above, the stored value comes from a + * runtime source (skb->len), so it cannot be constant-folded and + * clang always emits a genuine BPF_STX (register store) regardless + * of the target cpu version. + */ + switch (access_size) { + case 1: + val.data_1 = (__u8)skb->len; + break; + case 2: + val.data_2 = (__u16)skb->len; + break; + case 4: + val.data_4 = (__u32)skb->len; + break; + case 8: + val.data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = (__u8)skb->len; + break; + case 2: + val->data_2 = (__u16)skb->len; + break; + case 4: + val->data_4 = (__u32)skb->len; + break; + case 8: + val->data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val.data_1); + break; + case 2: + __sink(val.data_2); + break; + case 4: + __sink(val.data_4); + break; + case 8: + __sink(val.data_8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val->data_1); + break; + case 2: + __sink(val->data_2); + break; + case 4: + __sink(val->data_4); + break; + case 8: + __sink(val->data_8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sink(val->data_4); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sink(val.data_4); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val.data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val.data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val->data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val->data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val.data_8, 8); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val->data_8, 8); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +bool skip_load_acq_store_rel_tests SEC(".data") = 0; + +SEC("tcx/ingress") +int load_acquire_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val.data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val.data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val.data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val.data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int load_acquire_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val->data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val->data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val->data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val->data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val.data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val.data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val.data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val.data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val->data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val->data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val->data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val->data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} +#else +bool skip_load_acq_store_rel_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int verifier_paths_stack_and_non_stack(struct __sk_buff *skb) +{ + struct kasan_test_val stack_val = {}; + struct kasan_test_val *val; + void *ptr; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + if (access_size) + ptr = val; + else + ptr = &stack_val; + + bpf_kfunc_kasan_poison(val, sizeof(*val)); + *(__u8 *)ptr = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(*val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_oob(struct __sk_buff *skb) +{ + struct bpf_testmod_oob *val; + struct kasan_test_val volatile tmp; + + val = bpf_testmod_oob_alloc(); + if (!val) + return 0; + + switch (access_size) { + case 1: + tmp.data_1 = (__u8)val->redzone_1; + break; + case 2: + tmp.data_2 = (__u16)val->redzone_2; + break; + case 4: + tmp.data_4 = (__u32)val->redzone_4; + break; + case 8: + tmp.data_8 = (__u64)val->redzone_8; + break; + } + bpf_testmod_oob_free(val); + return tmp.data_1; +} + +SEC("tcx/ingress") +int ldx_self_alias_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + __u64 addr; + + bpf_kfunc_kasan_poison(&val, sizeof(val)); + /* + * Check that a stack access with dst_reg == src_reg is correctly + * flagged as stack-only access + */ + addr = (__u64)&val; + asm volatile( + "r1 = %0\n" + "r1 = *(u64 *)(r1 + 0)\n" + : + : "r"(addr) + : "r1", "memory"); + bpf_kfunc_kasan_unpoison(&val, sizeof(val)); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kasan_harden.c b/tools/testing/selftests/bpf/progs/kasan_harden.c new file mode 100644 index 000000000000..8c9eb203419c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan_harden.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include +#include + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_blinded(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + val->data_1 = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 2380b6cbdead..f798bbbb4d13 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -75,6 +75,16 @@ union bpf_testmod_union_arg_2 { struct bpf_testmod_struct_arg_2 arg; }; +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + __bpf_hook_start(); noinline int @@ -336,6 +346,47 @@ __bpf_kfunc void bpf_kfunc_put_default_trusted_ptr_test(struct prog_test_member */ } +#ifdef CONFIG_BPF_JIT_KASAN + +extern void kasan_poison(const void *addr, size_t size, u8 value, bool init); + +#define KASAN_SLAB_FREE 0xFB + +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, KASAN_SLAB_FREE, false); +} + +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, 0x00, false); +} +#else +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) { } +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) { } +#endif + +__bpf_kfunc struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) +{ + struct bpf_testmod_oob *p; + + /* + * Only allocate size of data (and so, voluntarily use kmalloc + * instead of kmalloc_obj), not the rest of the structure, so + * that programs under test trying to access the rest of the + * structure trigger OoB accesses + */ + p = kmalloc(sizeof(p->data), GFP_ATOMIC); + if (!p) + return NULL; + return p; +} + +__bpf_kfunc void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) +{ + kfree(oob); +} + __bpf_kfunc struct bpf_testmod_ctx * bpf_testmod_ctx_create(int *err) { @@ -869,6 +920,10 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); +BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) +BTF_ID_FLAGS(func, bpf_kfunc_kasan_unpoison) +BTF_ID_FLAGS(func, bpf_testmod_oob_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_testmod_oob_free, KF_RELEASE) BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) BTF_ID_LIST(bpf_testmod_dtor_ids) From f80a09c486e7c2a0e4ca43dbff7570df668484c5 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Sat, 5 Sep 2026 08:58:55 +0000 Subject: [PATCH 088/660] riscv, bpf: Remap BPF_REG_0 and RV_REG_TCC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To prepare for supporting bpf stack arguments and unifying the calling convention between bpf2bpf calls and kfunc calls, remap BPF_REG_0 to t6 and RV_REG_TCC to t5. In the riscv abi, a0-a7 are designated for function argument passing. Freeing a5 and a6 allows bpf2bpf calls to pass extra arguments (6th to 8th) via a5-a7 consistently with kfuncs. Signed-off-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-2-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 10 +++++----- arch/riscv/net/bpf_timed_may_goto.S | 8 ++++---- .../testing/selftests/bpf/progs/verifier_jit_inline.c | 2 +- tools/testing/selftests/bpf/progs/verifier_ldsx.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index e7378be171a9..fc1c8c5480b6 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -23,11 +23,11 @@ /* fentry and TCC init insns will be skipped on tailcall */ #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) -#define RV_REG_TCC RV_REG_A6 +#define RV_REG_TCC RV_REG_T5 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { - [BPF_REG_0] = RV_REG_A5, + [BPF_REG_0] = RV_REG_T6, [BPF_REG_1] = RV_REG_A0, [BPF_REG_2] = RV_REG_A1, [BPF_REG_3] = RV_REG_A2, @@ -47,13 +47,13 @@ static const int pt_regmap[] = { [RV_REG_A2] = offsetof(struct pt_regs, a2), [RV_REG_A3] = offsetof(struct pt_regs, a3), [RV_REG_A4] = offsetof(struct pt_regs, a4), - [RV_REG_A5] = offsetof(struct pt_regs, a5), [RV_REG_S1] = offsetof(struct pt_regs, s1), [RV_REG_S2] = offsetof(struct pt_regs, s2), [RV_REG_S3] = offsetof(struct pt_regs, s3), [RV_REG_S4] = offsetof(struct pt_regs, s4), [RV_REG_S5] = offsetof(struct pt_regs, s5), [RV_REG_T0] = offsetof(struct pt_regs, t0), + [RV_REG_T6] = offsetof(struct pt_regs, t6), }; enum { @@ -239,7 +239,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); + emit_addiw(RV_REG_A0, regmap[BPF_REG_0], 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } @@ -1075,7 +1075,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); if (save_ret) - stack_size += 16; /* Save both A5 (BPF R0) and A0 */ + stack_size += 16; /* Save both RV_REG_T6 (BPF R0) and RV_REG_A0 */ retval_off = stack_size; stack_size += nr_arg_slots * 8; diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 02c637d87420..fcf850f244c9 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,7 +10,7 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. */ @@ -20,8 +20,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S s0, 6*SZREG(sp) addi s0, sp, 8*SZREG - /* Save BPF registers R0-R5 (a5, a0-a4) */ - REG_S a5, 5*SZREG(sp) + /* Save BPF registers R0-R5 (t6, a0-a4) */ + REG_S t6, 5*SZREG(sp) REG_S a0, 4*SZREG(sp) REG_S a1, 3*SZREG(sp) REG_S a2, 2*SZREG(sp) @@ -38,7 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a2, 2*SZREG(sp) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) - REG_L a5, 5*SZREG(sp) + REG_L t6, 5*SZREG(sp) REG_L s0, 6*SZREG(sp) REG_L ra, 7*SZREG(sp) diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 02e562f56f9d..5b92da2ab75c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -11,7 +11,7 @@ __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 __jited(" mrs x8, SP_EL0") __arch_riscv64 -__jited(" mv a5, tp") +__jited(" mv t6, tp") __arch_loongarch __jited(" move $a5, $tp") int inline_bpf_get_current_task(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 7e25e1b8cd0b..49e8cd4bb4f9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -289,11 +289,11 @@ __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") __arch_riscv64 -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lw s3, 0x10(t2)") -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lh s3, 0x18(t2)") -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lb s3, 0x20(t2)") __jited("add t2, a0, s7") __jited("lw s4, 0x10(t2)") From ba754185f90291a0a164ec9adf7ba19737f18d3f Mon Sep 17 00:00:00 2001 From: Feng Jiang Date: Sat, 5 Sep 2026 08:58:56 +0000 Subject: [PATCH 089/660] riscv, bpf: Add BPF stack arguments support for RV64 JIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement bpf_jit_supports_stack_args() on RV64 JIT to allow bpf subprogs and kfuncs to pass and receive more than 5 arguments. In the riscv abi, the first 8 arguments are passed in registers a0 to a7, and arguments 9+ reside on the stack. To align bpf stack arguments with this calling convention and unify bpf2bpf calls with kfuncs, map the first 3 bpf stack arguments (6th to 8th) directly to a5 to a7, and store or load the remaining arguments (9th+) at SP or FP. Reserve outgoing stack space in the prologue accordingly when stack_arg_cnt exceeds 3. In addition, update kfunc argument sign extension to handle all 8 register arguments as well as any arguments passed on the stack. Co-developed-by: Pu Lehui Signed-off-by: Feng Jiang Signed-off-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-3-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit.h | 4 ++ arch/riscv/net/bpf_jit_comp64.c | 78 +++++++++++++++++++++++++++++---- arch/riscv/net/bpf_jit_core.c | 7 +++ 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 419b9d795f2a..5c4f53060c91 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -12,6 +12,9 @@ #include #include +#define RV_MAX_REG_ARGS 8 +#define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS) + /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) @@ -82,6 +85,7 @@ struct rv_jit_context { unsigned long flags; int stack_size; int tcc_offset; + int stack_arg_sz; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index fc1c8c5480b6..7f84447a31ff 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -15,7 +15,6 @@ #include #include "bpf_jit.h" -#define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) /* imm that allows emit_imm to emit max count insns */ @@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx) +{ + int idx = off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(rd, RV_REG_A5 + idx, ctx); + return; + } + + emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx); +} + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) { emit_imm(RV_REG_T1, imm, ctx); @@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_imm(RV_REG_A5 + idx, imm, ctx); + return; + } + + emit_imm(RV_REG_T1, imm, ctx); + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx); +} + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) { if (is_12b_int(off)) { @@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(RV_REG_A5 + idx, rs, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx); +} + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, struct rv_jit_context *ctx) { @@ -1824,11 +1860,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -EINVAL; for (idx = 0; idx < fm->nr_args; idx++) { - u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; + u8 arg_sz = fm->arg_size[idx]; - if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) - return -EINVAL; + if (arg_sz == 8 || (arg_sz != 4 && !sign)) + continue; + + if (idx < RV_MAX_REG_ARGS) { + if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx)) + return -EINVAL; + } else { + emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx); + if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx)) + return -EINVAL; + emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx); + } } } @@ -1927,7 +1973,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rs = RV_REG_T2; } - emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); + if (is_stack_arg_ldx(insn)) + emit_stack_arg_ldx(rd, off, ctx); + else + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); ret = add_exception_handler(insn, rd, ctx); if (ret) @@ -1957,7 +2006,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T3; } - emit_st(rd, off, imm, BPF_SIZE(code), ctx); + if (is_stack_arg_st(insn)) + emit_stack_arg_st(off, imm, ctx); + else + emit_st(rd, off, imm, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -1979,7 +2031,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T2; } - emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + if (is_stack_arg_stx(insn)) + emit_stack_arg_stx(off, rs, ctx); + else + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -2029,9 +2084,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { - int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -2190,3 +2245,8 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_supports_stack_args(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index cbfcd287ea16..2fb0b4e198b9 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -48,6 +48,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; + u16 stack_arg_cnt; if (!prog->jit_requested) return prog; @@ -71,6 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + + stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); + /* First 3 stack args in regs, rest on stack */ + ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ? + round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0; + ctx->prog = prog; ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) From 7687f1cbbaf8f17cddce4dd5c23a610c5c9a17f3 Mon Sep 17 00:00:00 2001 From: Feng Jiang Date: Sat, 5 Sep 2026 08:58:57 +0000 Subject: [PATCH 090/660] selftests/bpf: Enable stack argument tests for RV64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the stack argument selftests for RV64. The riscv guard is restricted to __riscv_xlen == 64 since stack arguments are only supported by the RV64 JIT. Keep the __BPF_FEATURE_STACK_ARGUMENT gate, defined by clang with BPF stack argument support [1]. [1] https://github.com/llvm/llvm-project/pull/189060 Signed-off-by: Feng Jiang Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-4-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c | 3 ++- .../selftests/bpf/progs/btf__verifier_stack_arg_order.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg_kfunc.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg_precision.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_stack_arg.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c | 3 ++- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c index 8d38aafe66a2..ab12397a9b52 100644 --- a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c @@ -4,7 +4,8 @@ #include #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size) diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index 99bc115f8380..a980a3a399d3 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -3,7 +3,8 @@ #include #include -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f) diff --git a/tools/testing/selftests/bpf/progs/stack_arg.c b/tools/testing/selftests/bpf/progs/stack_arg.c index 944e3bb603e7..3e9ed37c57bb 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg.c +++ b/tools/testing/selftests/bpf/progs/stack_arg.c @@ -21,7 +21,8 @@ struct { int timer_result; -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c index 345f2da2e361..b6b1a8aa7f44 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c @@ -6,7 +6,8 @@ #include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index ce0301a41fa9..7baa9c1c8e31 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -6,7 +6,8 @@ #include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) /* Force kfunc extern BTF generation for inline asm call below. diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 51d22faf4559..f6a4a6aa0e61 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -12,7 +12,8 @@ struct { __type(value, long long); } map_hash_8b SEC(".maps"); -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index 8e4325273b3d..57f22691744a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,7 +5,8 @@ #include #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used __naked From e62c5b97d6c4c657d6fad7dd3523fda33a936979 Mon Sep 17 00:00:00 2001 From: Mingpei CAO Date: Tue, 8 Sep 2026 16:49:19 +0000 Subject: [PATCH 091/660] libbpf: Fix array comparison in BTF dedup btf_dedup_identical_types() reads both array descriptors from t1, skipping comparisons of their referenced types. This can incorrectly merge distinct structs and corrupt CO-RE relocation metadata. Read the second descriptor from t2. Fixes: 62e23f183839 ("libbpf: Improve BTF dedup handling of "identical" BTF types") Closes: https://lore.kernel.org/bpf/CABzjXVz3iBuump-pXFkefZ5v+2uu4fG61zqRyWD4xmz3PuBycw@mail.gmail.com/ Signed-off-by: Mingpei CAO Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260908164920.108074-2-caomingpei@gmail.com --- tools/lib/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index c783359977b4..908bd344229d 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -4831,7 +4831,7 @@ recur: return false; a1 = btf_array(t1); - a2 = btf_array(t1); + a2 = btf_array(t2); if (a1->index_type != a2->index_type && !btf_dedup_identical_types(d, a1->index_type, a2->index_type, depth - 1)) From 208637af0dc2ddad0e0d96dbd02b961a2cb6fd8a Mon Sep 17 00:00:00 2001 From: Mingpei CAO Date: Tue, 8 Sep 2026 16:49:20 +0000 Subject: [PATCH 092/660] selftests/bpf: Test array element comparison in BTF dedup Exercise recursive array comparison with different and identical element definitions using the existing BTF dedup tests. Add a separate case with the same index type to isolate the element-type comparison. Check that distinct containers remain separate and identical types are deduplicated. Both negative cases fail before the fix and pass afterwards; the positive case passes both before and after the fix. Suggested-by: Alan Maguire Signed-off-by: Mingpei CAO Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260908164920.108074-3-caomingpei@gmail.com --- tools/testing/selftests/bpf/prog_tests/btf.c | 126 +++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 67b9015cbd98..df6ad38d287d 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -7062,6 +7062,132 @@ static struct btf_dedup_test dedup_tests[] = { BTF_STR_SEC("\0int\0long int"), }, }, +{ + .descr = "dedup: array element comparison", + .input = { + .raw_types = { + /* signed int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + /* signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + /* duplicate signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + /* unsigned int[1] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [5] */ + /* struct s { signed int a[1]; signed int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { signed int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: array element comparison with same index type", + .input = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [5] */ + /* struct s { int a[1]; int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: identical array element comparison", + .input = { + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* struct container { struct elem first[1]; struct elem second[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + /* struct elem[1] */ + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + /* struct elem { int x; } */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + /* duplicate struct container */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(3), 6, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 7, 32), + /* duplicate struct elem[1] */ + BTF_TYPE_ARRAY_ENC(8, 1, 1), /* [6] */ + BTF_TYPE_ARRAY_ENC(9, 1, 1), /* [7] */ + /* duplicate struct elem */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [8] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [9] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, +}, { .descr = "dedup: struct example #1", /* From a4c6292256f1172aaed795af9e0dbd2aab6de6ec Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Tue, 8 Sep 2026 10:25:05 -0700 Subject: [PATCH 093/660] libbpf: Defer arena map size check to load time init_arena_map_data() used the build-host page size while it opened an object. This could reject skeleton generation when the build host and target use different page sizes. Validate the size during load, before libbpf calculates the arena data offset. This uses the running kernel page size and rejects an undersized map before relocation. This problem manifested when cross compiling BPF selftests for arm64 (64K page) on x86 (4K page). In this setup skeleton generation fails. Signed-off-by: Mykyta Yatsenko Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260908-libbpf_arena_thing-v1-1-0092c0e0d91b@meta.com --- tools/lib/bpf/libbpf.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index c036e8a91ed8..8da5181c8e8b 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -3018,20 +3018,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, } static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, - const char *sec_name, int sec_idx, void *data, size_t data_sz) { - const long page_sz = sysconf(_SC_PAGE_SIZE); - const size_t data_alloc_sz = roundup(data_sz, page_sz); - size_t mmap_sz; - - mmap_sz = bpf_map_mmap_sz(map); - if (data_alloc_sz > mmap_sz) { - pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", - sec_name, mmap_sz, data_sz); - return -E2BIG; - } - obj->arena_data = malloc(data_sz); if (!obj->arena_data) return -ENOMEM; @@ -3107,8 +3095,7 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, obj->arena_map_idx = i; if (obj->efile.arena_data) { - err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, - obj->efile.arena_data->d_buf, + err = init_arena_map_data(obj, map, obj->efile.arena_data->d_buf, obj->efile.arena_data->d_size); if (err) return err; @@ -7489,12 +7476,20 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat bpf_object__sort_relos(obj); } - /* place globals at the end of the arena (if supported) */ - if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) { + if (obj->arena_map_idx >= 0) { struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx]; + size_t data_sz = roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE)); + size_t mmap_sz = bpf_map_mmap_sz(arena_map); - obj->arena_data_off = bpf_map_mmap_sz(arena_map) - - roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE)); + if (data_sz > mmap_sz) { + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", + arena_map->name, mmap_sz, obj->arena_data_sz); + return -E2BIG; + } + + /* place globals at the end of the arena (if supported) */ + if (kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) + obj->arena_data_off = mmap_sz - data_sz; } /* Before relocating calls pre-process relocations and mark From 88c9c6d0a766dd146545990053e8a04422ccb367 Mon Sep 17 00:00:00 2001 From: Sanghyun Park Date: Tue, 1 Sep 2026 10:48:29 +0900 Subject: [PATCH 094/660] bpf: Fix program BTF use-after-free in sleepable programs Object kfunc calls embed metadata owned by the program BTF. A sleepable program can remain active under Tasks Trace RCU after its last reference is dropped, while program teardown releases the BTF through ordinary RCU. The invocation can then dereference freed metadata in bpf_obj_new(). Move btf_put() into __bpf_prog_put_rcu(), the callback that frees the program. When teardown is deferred, that callback runs after the program's own grace period (Tasks Trace RCU for sleepable programs and ordinary RCU otherwise), so the BTF outlives every active invocation. The non-deferred path invokes the callback synchronously, so load-error cleanup stays direct. Fixes: 958cf2e273f0 ("bpf: Introduce bpf_obj_new") Signed-off-by: Sanghyun Park Signed-off-by: Andrii Nakryiko Reviewed-by: Sun Jian Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260901014829.3504342-2-sanghyun.park.cnu@gmail.com --- kernel/bpf/syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c7bc9ba9b331..def57bddb092 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2441,6 +2441,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) { struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + btf_put(aux->btf); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2451,7 +2452,6 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { bpf_prog_kallsyms_del_all(prog); - btf_put(prog->aux->btf); module_put(prog->aux->mod); kvfree(prog->aux->jited_linfo); kvfree(prog->aux->linfo); From 34b9bde8eea76c7b28cb64cd1bc06b06f3dd18fb Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 3 Sep 2026 16:03:51 +0100 Subject: [PATCH 095/660] selftests/bpf: test_xsk: Fix spelling mistake "conigure" -> "configure" There is a spelling mistake in an assert message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Andrii Nakryiko Reviewed-by: Bastien Curutchet Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260903150351.12725-1-colin.i.king@gmail.com --- tools/testing/selftests/bpf/prog_tests/xsk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/xsk.c b/tools/testing/selftests/bpf/prog_tests/xsk.c index 6e2f63ee2a6c..772f3dd7ce64 100644 --- a/tools/testing/selftests/bpf/prog_tests/xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/xsk.c @@ -75,7 +75,7 @@ static void test_xsk(const struct test_spec *test_to_run, enum test_mode mode) if (!ASSERT_OK_PTR(ifobj_rx, "create ifobj_rx")) goto delete_tx; - if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "conigure ifobj")) + if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "configure ifobj")) goto delete_rx; ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); From cf95f22a06b22fc5ecf02bd60faece12e6f2d62b Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 7 Sep 2026 00:26:56 +0200 Subject: [PATCH 096/660] bpf, docs: Fix two struct names in btf.rst and kfuncs.rst struct btf_variable and struct bpf_cpmuask do not exist; the names are struct btf_var and struct bpf_cpumask. Use them. Fixes: f063c889c945 ("bpf: add specification for BTF Var and DataSec kinds") Fixes: 027bdec89364 ("bpf/docs: Document the nocast aliasing behavior of ___init") Signed-off-by: Karl Mehltretter Signed-off-by: Andrii Nakryiko Reviewed-by: Randy Dunlap Link: https://lore.kernel.org/bpf/20260906222656.3459-1-kmehltretter@gmail.com --- Documentation/bpf/btf.rst | 2 +- Documentation/bpf/kfuncs.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 3f05f17990ad..004aa1058d85 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -416,7 +416,7 @@ If the function has variable arguments, the last parameter is encoded with * ``info.vlen``: 0 * ``type``: the type of the variable -``btf_type`` is followed by a single ``struct btf_variable`` with the +``btf_type`` is followed by a single ``struct btf_var`` with the following data:: struct btf_var { diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index ebe37c1fe9d2..6c2c048dccef 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -536,7 +536,7 @@ For example, for the following type definition: The verifier would allow a ``struct bpf_cpumask *`` to be passed to a kfunc taking a ``cpumask_t *`` (which is a typedef of ``struct cpumask *``). For -instance, both ``struct cpumask *`` and ``struct bpf_cpmuask *`` can be passed +instance, both ``struct cpumask *`` and ``struct bpf_cpumask *`` can be passed to bpf_cpumask_test_cpu(). In some cases, this type-aliasing behavior is not desired. ``struct From c1ff425d625eb2a4d2967e9889b85f202f24eb5d Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Sun, 6 Sep 2026 22:33:58 +0000 Subject: [PATCH 097/660] bpf: btf: Make alloc_obj_fields array const The alloc_obj_fields array is never modified. Make the array itself const, not just the strings it points to. Signed-off-by: Bradley Morgan Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260906223358.25717-1-brads@mainlining.org --- kernel/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 31057c8f3a7c..5fdb6ee2ca92 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5797,7 +5797,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env) return btf_check_sec_info(env, btf_data_size); } -static const char *alloc_obj_fields[] = { +static const char * const alloc_obj_fields[] = { "bpf_spin_lock", "bpf_list_head", "bpf_list_node", From 6523af719a2156e1827083c3af4ba718ef862366 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:53 -0700 Subject: [PATCH 098/660] bpf: Pass call metadata through shared argument checks Helper, kfunc, and global subprogram argument paths supply callee identity and state differently. check_reg_type() receives the call name separately from bpf_call_arg_meta, while process_dynptr_func() receives the call name, referenced-object state, and dynptr state as separate arguments. Populate meta->func_name for helper and global subprogram calls, keep one metadata object for the entire global subprogram argument check, and have both shared checking functions obtain their name and state from the metadata. This gives dynptr validation one interface for every call kind in preparation for unifying their argument-checking paths. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-2-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 67 +++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9e79750e2480..a45e0478f23c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7663,11 +7663,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u /* * Validate dynptr arguments for helper, kfunc and subprog. * - * @dynptr is both input and output. It is populated when the argument is - * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) - * and consumed when the argument is expecting to be an initialized dynptr. - * @parent_id is used to track the referenced parent object (e.g., file or skb in - * qdisc program) when constructing a dynptr. + * @meta carries the dynptr and referenced-object state. The dynptr is populated + * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that + * will be constructed) and consumed when the argument is expected to be an + * initialized dynptr. The reference tracks the parent object (e.g., file or skb + * in qdisc program) when constructing a dynptr. * * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. @@ -7684,9 +7684,8 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, const char *call_name, - enum bpf_arg_type arg_type, - struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) + argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta) { int spi, err = 0; @@ -7695,7 +7694,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, + env, insn_idx, argno, meta->func_name, "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); @@ -7736,7 +7735,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, + &meta->ref_obj, &meta->dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7766,7 +7766,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(expected_type), reg_arg_name(env, argno)); bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, + env, insn_idx, argno, meta->func_name, "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", dynptr_type_str(actual_type), dynptr_type_str(expected_type)); @@ -7785,11 +7785,9 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat reg = &state->stack[spi].spilled_ptr; } - if (dynptr) { - dynptr->type = reg->dynptr.type; - dynptr->id = reg->id; - dynptr->parent_id = reg->parent_id; - } + meta->dynptr.type = reg->dynptr.type; + meta->dynptr.id = reg->id; + meta->dynptr.parent_id = reg->parent_id; } return err; } @@ -8382,7 +8380,7 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type, const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta, const char *call_name) + struct bpf_call_arg_meta *meta) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; @@ -8435,7 +8433,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); accepted = bpf_diag_expected_reg_types(env, compatible->types, i); - bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, "Pass a value with one of the accepted pointer or scalar types for this call.", "it has type %s, but this argument accepts %s", actual, accepted); @@ -8449,7 +8447,7 @@ found: if (!(arg_type & MEM_RDONLY)) { verbose(env, "%s() may write into memory pointed by %s type=%s\n", - func_id_name(meta->func_id), + meta->func_name, reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } @@ -8476,7 +8474,7 @@ found: verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); bpf_diag_call_arg( - env, env->insn_idx, argno, call_name, + env, env->insn_idx, argno, meta->func_name, "the pointer may be NULL, but this call requires a non-NULL pointer", "Add a NULL check and make the call only on the non-NULL path."); return -EACCES; @@ -8869,8 +8867,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, - func_id_name(meta->func_id)); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta); if (err) return err; @@ -8882,9 +8879,9 @@ skip_type_check: if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_id_name(meta->func_id), reg_arg_name(env, argno)); + meta->func_name, reg_arg_name(env, argno)); bpf_diag_call_arg( - env, insn_idx, argno, func_id_name(meta->func_id), + env, insn_idx, argno, meta->func_name, "release helpers require a value that owns a live resource returned by a matching acquire helper", "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); return -EINVAL; @@ -9015,8 +9012,7 @@ skip_type_check: true, meta, NULL); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), - arg_type, &meta->ref_obj, &meta->dynptr); + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; break; @@ -9763,12 +9759,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; - struct ref_obj_desc ref_obj = {}; const struct btf_param *args; const struct btf_type *func, *func_proto; + struct bpf_call_arg_meta meta; u32 i; int ret, err; + /* Leave btf and func_id zero: this is neither a helper nor a kfunc. */ + memset(&meta, 0, sizeof(meta)); + meta.func_name = bpf_subprog_name(env, subprog); + ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { @@ -9852,12 +9852,10 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, return ret; ret = process_dynptr_func(env, reg, argno, env->insn_idx, - bpf_subprog_name(env, subprog), arg->arg_type, - &ref_obj, NULL); + arg->arg_type, &meta); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { - struct bpf_call_arg_meta meta; int err; if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { @@ -9867,9 +9865,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; } - memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, - bpf_subprog_name(env, subprog)); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; @@ -11008,6 +11004,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.func_name = func_id_name(func_id); meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { @@ -13177,8 +13174,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); } - ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, - dynptr_arg_type, &meta->ref_obj, &meta->dynptr); + ret = process_dynptr_func(env, reg, argno, insn_idx, + dynptr_arg_type, meta); if (ret < 0) return ret; break; From 47f4215b8b5486fd85dd7993ae12a989a7ac8db6 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:54 -0700 Subject: [PATCH 099/660] bpf: Address check_func_arg() arguments by argno check_func_arg() derives every argument from BPF_REG_1 + arg. That works for helpers, which have at most five arguments, but kfunc arguments beyond the fifth are passed on the stack. Switch to the argno_t addressing used by check_kfunc_args(): obtain argument state with get_func_arg_reg(), use argno for diagnostics, and call check_reg_arg() only when the argument is held in a register. This prepares check_func_arg() to be shared with kfunc verification. There is no functional change for helpers. reg_arg_name() renders their register arguments as R%d, so the updated diagnostics remain unchanged. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-3-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a45e0478f23c..d5e885435bb6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8815,10 +8815,12 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, int insn_idx) { const struct bpf_func_proto *fn = meta->fn; - u32 regno = BPF_REG_1 + arg; - struct bpf_reg_state *reg = reg_state(env, regno); + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *regs = cur_regs(env); + argno_t argno = argno_from_arg(arg + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, arg); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_reg(regno); + int regno = reg_from_argno(argno); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; @@ -8827,14 +8829,16 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (arg_type == ARG_DONTCARE) return 0; - err = check_reg_arg(env, regno, SRC_OP); - if (err) - return err; + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } if (arg_type == ARG_ANYTHING) { - if (is_pointer_value(env, regno)) { - verbose(env, "R%d leaks addr into helper function\n", - regno); + if (__is_pointer_value(env->allow_ptr_leaks, reg)) { + verbose(env, "%s leaks addr into helper function\n", + reg_arg_name(env, argno)); return -EACCES; } return 0; @@ -8954,7 +8958,7 @@ skip_type_check: break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { - verbose(env, "Helper has invalid btf_id in R%d\n", regno); + verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); return -EACCES; } meta->ret_btf = reg->btf; @@ -8991,7 +8995,7 @@ skip_type_check: * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + err = check_mem_reg(env, reg, argno, fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; @@ -9000,14 +9004,14 @@ skip_type_check: } break; case ARG_MEM_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, + err = check_mem_size_reg(env, get_func_arg_reg(caller, regs, arg - 1), reg, + argno_from_arg(arg), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); break; case ARG_MEM_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, + err = check_mem_size_reg(env, get_func_arg_reg(caller, regs, arg - 1), reg, + argno_from_arg(arg), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, true, meta, NULL); break; From d18bcf4537e028c532a5d6838605df070cc1c6fb Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:55 -0700 Subject: [PATCH 100/660] bpf: Only compare func_id against BPF_FUNC_* for helper calls Helpers are identified by enum bpf_func_id, while kfuncs are identified by a BTF ID. Both values are stored in bpf_call_arg_meta::func_id, and a kfunc BTF ID can have the same numeric value as a BPF_FUNC_* constant. Later patches extend check_reg_type() and check_func_arg() to kfuncs. A bare func_id comparison in those common paths could then mistake a kfunc for a helper. Introduce is_helper_call(), which first excludes kfunc metadata through meta->btf, and use it for every BPF_FUNC_* comparison. This keeps helper-specific behavior out of the shared path from the start. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-4-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d5e885435bb6..5bf31fbdaac1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8176,6 +8176,16 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } +/* + * A kfunc is named by a BTF ID, which can take the same numeric value as an + * enum bpf_func_id. Only test meta->func_id against a BPF_FUNC_* once the call + * is known to be to a helper; meta->btf is set only for a kfunc. + */ +static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) +{ + return !meta->btf && meta->func_id == func_id; +} + static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type) @@ -8197,7 +8207,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } break; case BPF_MAP_TYPE_BLOOM_FILTER: - if (meta->func_id == BPF_FUNC_map_peek_elem) + if (is_helper_call(meta, BPF_FUNC_map_peek_elem)) *arg_type = ARG_PTR_TO_MAP_VALUE; break; default: @@ -8413,7 +8423,8 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~DYNPTR_TYPE_FLAG_MASK; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && + reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; } @@ -8467,7 +8478,7 @@ found: * allows bpf_sk_release to work for multiple socket types. */ bool strict_type_match = arg_type_is_release(arg_type) && - meta->func_id != BPF_FUNC_sk_release; + !is_helper_call(meta, BPF_FUNC_sk_release); if (type_may_be_null(reg->type) && (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { @@ -8488,7 +8499,7 @@ found: arg_btf_id = compatible->btf_id; } - if (meta->func_id == BPF_FUNC_kptr_xchg) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) { if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) return -EACCES; } else { @@ -8519,13 +8530,14 @@ found: case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: - if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && - meta->func_id != BPF_FUNC_kptr_xchg) { + if (!is_helper_call(meta, BPF_FUNC_spin_lock) && + !is_helper_call(meta, BPF_FUNC_spin_unlock) && + !is_helper_call(meta, BPF_FUNC_kptr_xchg)) { verifier_bug(env, "unimplemented handling of MEM_ALLOC"); return -EFAULT; } /* Check if local kptr in src arg matches kptr in dst arg */ - if (meta->func_id == BPF_FUNC_kptr_xchg) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) { int regno = reg_from_argno(argno); if (regno == BPF_REG_2 && @@ -8948,7 +8960,7 @@ skip_type_check: * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads * the value buffer as an input rather than filling it. */ - if (meta->func_id == BPF_FUNC_map_peek_elem && + if (is_helper_call(meta, BPF_FUNC_map_peek_elem) && meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) meta->arg_raw_mem.regno = 0; @@ -8969,11 +8981,11 @@ skip_type_check: verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } - if (meta->func_id == BPF_FUNC_spin_lock) { + if (is_helper_call(meta, BPF_FUNC_spin_lock)) { err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; - } else if (meta->func_id == BPF_FUNC_spin_unlock) { + } else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) { err = process_spin_lock(env, reg, argno, 0); if (err) return err; From 0bddc59e33a44655e3402850c0c2dc27296bab09 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:56 -0700 Subject: [PATCH 101/660] bpf: Only compare func_id against kfunc BTF IDs for kfunc calls Helpers are identified by enum bpf_func_id, while kfuncs are identified by a BTF ID. Both values are stored in bpf_call_arg_meta::func_id, and a helper ID can have the same numeric value as a kfunc BTF ID. Introduce is_kfunc_call(), which first confirms that the metadata is for a kfunc through meta->btf, and use it for comparisons against the special kfunc BTF ID list. This complements is_helper_call() before later patches move these checks into paths shared by helpers and kfuncs. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-5-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 68 +++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5bf31fbdaac1..42d89db5df07 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8186,6 +8186,11 @@ static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_i return !meta->btf && meta->func_id == func_id; } +static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) +{ + return meta->btf && meta->func_id == btf_id; +} + static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type) @@ -12137,27 +12142,27 @@ static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]); } static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]); } static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]); } static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); } bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]); } static int @@ -12198,9 +12203,9 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) arg_type = KF_ARG_PTR_TO_CTX; else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) arg_type = KF_ARG_PTR_TO_CTX; @@ -12411,15 +12416,15 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; - if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { irq_save = true; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) kfunc_class = IRQ_LOCK_KFUNC; - } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) { irq_save = false; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) kfunc_class = IRQ_LOCK_KFUNC; } else { verifier_bug(env, "unknown irq flags kfunc"); @@ -13111,7 +13116,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return -EINVAL; } - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); if (ret < 0) return -EINVAL; @@ -13168,17 +13173,17 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (is_kfunc_arg_uninit(btf, &args[i])) dynptr_arg_type |= MEM_UNINIT; - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) { dynptr_arg_type |= DYNPTR_TYPE_SKB; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) { dynptr_arg_type |= DYNPTR_TYPE_XDP; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) { dynptr_arg_type |= DYNPTR_TYPE_SKB_META; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file])) { dynptr_arg_type |= DYNPTR_TYPE_FILE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) { dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]) && (dynptr_arg_type & MEM_UNINIT)) { enum bpf_dynptr_type parent_type = meta->dynptr.type; @@ -13197,7 +13202,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } case KF_ARG_PTR_TO_ITER: - if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new])) { if (!check_css_task_iter_allowlist(env)) { verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); return -EINVAL; @@ -13578,11 +13583,12 @@ check_ok: if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) return -EFAULT; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) flags |= PROCESS_SPIN_LOCK; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || + is_kfunc_call(meta, + special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) flags |= PROCESS_LOCK_IRQ; ret = process_spin_lock(env, reg, argno, flags); if (ret < 0) @@ -13925,12 +13931,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct btf_field *field = meta->arg_rbtree_root.field; mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); - } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->ret_btf_id; - } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) { ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); if (!ret_t) { verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", @@ -13950,8 +13956,8 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); return -EINVAL; } - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || - meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) { enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); mark_reg_known_zero(env, regs, BPF_REG_0); @@ -13966,7 +13972,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) { regs[BPF_REG_0].type |= MEM_RDONLY; } else { /* this will set env->seen_direct_write to true */ From 6db166056c25f8091e14e15b132a795676fe7f7e Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:57 -0700 Subject: [PATCH 102/660] bpf: Clarify unused and scalar function argument types ARG_DONTCARE is the zero-valued terminator for the fixed-size helper argument array rather than an argument whose value is ignored. Rename it to ARG_UNUSED and stop helper argument iteration explicitly when it is encountered. Commit c3fd8e5fd100 ("bpf: Reject non-scalar bpf_loop iteration counts") introduced ARG_SCALAR for arguments that must be scalars. Reuse it for integer and enum global subprogram arguments, which already require SCALAR_VALUE despite being classified as ARG_ANYTHING, and for exception callback arguments. This leaves ARG_ANYTHING with its legacy helper behavior of accepting pointers when pointer leaks are allowed. Also pass ARG_PTR_TO_MEM when checking a global subprogram memory argument instead of using the prototype terminator as a placeholder. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-6-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 +- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 25 ++++++++++++------------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e80963971f68..dcbf8cea45d6 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -874,7 +874,7 @@ enum bpf_type_flag { /* function argument constraints */ enum bpf_arg_type { - ARG_DONTCARE = 0, /* unused argument in helper function */ + ARG_UNUSED = 0, /* unused argument; terminates argument iteration */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5fdb6ee2ca92..c693f2c345f2 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8244,7 +8244,7 @@ skip_pointer: return -EINVAL; } if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_ANYTHING; + sub->args[i].arg_type = ARG_SCALAR; continue; } if (!is_global) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 42d89db5df07..5d21bee45805 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8843,9 +8843,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 key_size; int err = 0; - if (arg_type == ARG_DONTCARE) - return 0; - if (regno >= 0) { err = check_reg_arg(env, regno, SRC_OP); if (err) @@ -9356,7 +9353,7 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; if (!arg_type_is_raw_mem(fn->arg_type[i])) continue; @@ -9406,7 +9403,7 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; @@ -9429,7 +9426,7 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; @@ -9447,7 +9444,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) @@ -9818,7 +9815,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); struct bpf_subprog_arg_info *arg = &sub->args[i]; - if (arg->arg_type == ARG_ANYTHING) { + if (arg->arg_type == ARG_SCALAR) { if (reg->type != SCALAR_VALUE) { bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; @@ -9842,7 +9839,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, return -EINVAL; } } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); + ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_MEM); if (ret < 0) return ret; if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, @@ -11029,6 +11026,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { + if (fn->arg_type[i] == ARG_UNUSED) + break; err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; @@ -12924,7 +12923,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me for (i = 0; i < nargs; i++) { struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); const struct btf_type *t, *ref_t, *resolve_ret; - enum bpf_arg_type arg_type = ARG_DONTCARE; + enum bpf_arg_type arg_type = ARG_UNUSED; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; @@ -17991,7 +17990,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, cs->is_void = fn->ret_type == RET_VOID; cs->num_params = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; cs->num_params++; } @@ -19795,7 +19794,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { + if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; @@ -19808,7 +19807,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) if (arg->arg_type == ARG_PTR_TO_CTX) { reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); - } else if (arg->arg_type == ARG_ANYTHING) { + } else if (arg->arg_type == ARG_SCALAR) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { From 5e7edfe68654ae12125918eec0507566405a371f Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:58 -0700 Subject: [PATCH 103/660] bpf: Unify kfunc argument kinds with enum bpf_arg_type check_kfunc_args() classifies arguments with enum kfunc_ptr_arg_type while check_func_arg() uses enum bpf_arg_type, yet both classifications are stored in bpf_func_proto::arg_type. The overlapping namespaces force the kfunc path to translate argument kinds before calling shared checks. Fold the kfunc kinds into enum bpf_arg_type. Reuse ARG_SCALAR and the existing pointer and memory kinds where their semantics match, map kfunc callbacks to ARG_PTR_TO_FUNC, and add enumerators for the remaining kfunc-only kinds. check_kfunc_args() can then carry one classification throughout verification. Preserving the original argument kind also lets check_func_arg_reg_off() derive the zero-offset requirement for ARG_PTR_TO_REFCOUNTED_KPTR directly. Remove its separate btf_id_fixed_off_ok parameter and wrapper, along with the now-empty translation switch in check_kfunc_args(). Keep the PTR_TO_CTX offset decision based on the base argument kind. The generated kind may carry flags such as PTR_MAYBE_NULL, while the removed translation discarded them. This retains the zero-offset requirement for context arguments regardless of such flags. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-7-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 14 +++ include/linux/bpf_verifier.h | 2 +- kernel/bpf/verifier.c | 223 ++++++++++++----------------------- 3 files changed, 89 insertions(+), 150 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index dcbf8cea45d6..ebfdf9d209f3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -909,6 +909,20 @@ enum bpf_arg_type { ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ + + ARG_CONST_SCALAR, /* scalar known at verification time */ + ARG_CONST_MEM_SIZE, /* ARG_MEM_SIZE that must be constant */ + ARG_PTR_TO_ALLOC_BTF_ID, /* pointer to an allocated object */ + ARG_PTR_TO_REFCOUNTED_KPTR, /* pointer to a refcounted local kptr */ + ARG_PTR_TO_ITER, /* pointer to an iterator */ + ARG_PTR_TO_LIST_HEAD, /* pointer to bpf_list_head */ + ARG_PTR_TO_LIST_NODE, /* pointer to bpf_list_node */ + ARG_PTR_TO_RB_ROOT, /* pointer to bpf_rb_root */ + ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ + ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ + ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ + ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 9727df5af83a..6b973b94ee75 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1591,7 +1591,7 @@ struct bpf_call_arg_meta { * verification logic * bpf_obj_drop/bpf_percpu_obj_drop * Record the local kptr type to be drop'd - * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type) * Record the local kptr type to be refcount_incr'd and use * arg_owning_ref to determine whether refcount_acquire should be * fallible diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d21bee45805..dc849e670876 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8579,10 +8579,9 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int __check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - bool btf_id_fixed_off_ok) +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -8638,12 +8637,15 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, - * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero unless the caller requires otherwise. - * var_off always must be 0 for PTR_TO_BTF_ID, hence we still - * need to do checks instead of returning. + * its fixed offset must be 0. bpf_refcount_acquire() returns the + * pointer it was given while incrementing the refcount at the + * refcount field offset, so it needs a zero offset too. In the + * other cases, fixed offset can be non-zero. var_off always must + * be 0 for PTR_TO_BTF_ID, hence we still need to do checks + * instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); + return __check_ptr_off_reg(env, reg, argno, + base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8651,7 +8653,7 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, * otherwise we may get modified ctx in tail called programs and * global subprogs (that may act as extension prog hooks). */ - if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) + if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) return 0; fallthrough; default: @@ -8659,13 +8661,6 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, } } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) -{ - return __check_func_arg_reg_off(env, reg, argno, arg_type, true); -} - static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -11889,34 +11884,6 @@ static void btf_member_path_str(const struct btf *btf, const struct btf_member_p } } -enum kfunc_ptr_arg_type { - KF_ARG_CONST_MEM_SIZE, - KF_ARG_MEM_SIZE, - KF_ARG_CONST, - KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, - KF_ARG_ANYTHING, - KF_ARG_PTR_TO_CTX, - KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ - KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ - KF_ARG_PTR_TO_DYNPTR, - KF_ARG_PTR_TO_ITER, - KF_ARG_PTR_TO_LIST_HEAD, - KF_ARG_PTR_TO_LIST_NODE, - KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ - KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_CALLBACK, - KF_ARG_PTR_TO_RB_ROOT, - KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_CONST_STR, - KF_ARG_CONST_MAP_PTR, - KF_ARG_PTR_TO_TIMER, - KF_ARG_PTR_TO_WORKQUEUE, - KF_ARG_PTR_TO_IRQ_FLAG, - KF_ARG_PTR_TO_RES_SPIN_LOCK, - KF_ARG_PTR_TO_TASK_WORK, - KF_ARG_PTR_TO_ARENA, -}; - enum special_kfunc_type { KF_bpf_obj_new_impl, KF_bpf_obj_new, @@ -12178,15 +12145,15 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, /* Scalar arguments are classified from their BTF suffix/name alone. */ if (btf_type_is_scalar(t)) { if (is_kfunc_arg_constant(meta->btf, &args[arg])) - return KF_ARG_CONST; + return ARG_CONST_SCALAR; if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) - return KF_ARG_CONST_MEM_SIZE; + return ARG_CONST_MEM_SIZE; if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) - return KF_ARG_MEM_SIZE; + return ARG_MEM_SIZE; if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) - return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; - return KF_ARG_ANYTHING; + return ARG_CONST_ALLOC_SIZE_OR_ZERO; + return ARG_SCALAR; } if (!btf_type_is_ptr(t)) { @@ -12200,48 +12167,48 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of - * arguments, we resolve it to a known kfunc_ptr_arg_type. + * arguments, we resolve it to a known bpf_arg_type. */ if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) - arg_type = KF_ARG_PTR_TO_CTX; + arg_type = ARG_PTR_TO_CTX; else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - arg_type = KF_ARG_PTR_TO_CTX; + arg_type = ARG_PTR_TO_CTX; else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + arg_type = ARG_PTR_TO_ALLOC_BTF_ID; else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_DYNPTR; + arg_type = ARG_PTR_TO_DYNPTR; else if (is_kfunc_arg_iter(meta, arg, &args[arg])) - arg_type = KF_ARG_PTR_TO_ITER; + arg_type = ARG_PTR_TO_ITER; else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_HEAD; + arg_type = ARG_PTR_TO_LIST_HEAD; else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_NODE; + arg_type = ARG_PTR_TO_LIST_NODE; else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_ROOT; + arg_type = ARG_PTR_TO_RB_ROOT; else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_NODE; + arg_type = ARG_PTR_TO_RB_NODE; else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CONST_STR; + arg_type = ARG_PTR_TO_CONST_STR; else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) - arg_type = KF_ARG_CONST_MAP_PTR; + arg_type = ARG_CONST_MAP_PTR; else if (is_kfunc_arg_map(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_BTF_ID; + arg_type = ARG_PTR_TO_BTF_ID; else if (is_kfunc_arg_wq(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_WORKQUEUE; + arg_type = ARG_PTR_TO_WORKQUEUE; else if (is_kfunc_arg_timer(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TIMER; + arg_type = ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TASK_WORK; + arg_type = ARG_PTR_TO_TASK_WORK; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + arg_type = ARG_PTR_TO_RES_SPIN_LOCK; else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CALLBACK; + arg_type = ARG_PTR_TO_FUNC; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { if (!bpf_jit_supports_arena_args()) { verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", @@ -12264,7 +12231,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * whether the JIT rebases it to the arena base or preserves NULL. * The common nullable path below records that verifier property. */ - arg_type = KF_ARG_PTR_TO_ARENA; + arg_type = ARG_PTR_TO_ARENA; } else if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { @@ -12274,10 +12241,10 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM; + arg_type = ARG_PTR_TO_MEM; } else if (btf_type_is_struct(ref_t)) - /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ - arg_type = KF_ARG_PTR_TO_BTF_ID; + /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ + arg_type = ARG_PTR_TO_BTF_ID; else { /* * Otherwise this is a fixed-size memory buffer supported by @@ -12290,7 +12257,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } if (is_kfunc_arg_nullable(meta->btf, &args[arg])) @@ -12922,13 +12889,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me */ for (i = 0; i < nargs; i++) { struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); + enum bpf_arg_type arg_type = meta->fn->arg_type[i]; const struct btf_type *t, *ref_t, *resolve_ret; - enum bpf_arg_type arg_type = ARG_UNUSED; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); - bool btf_id_fixed_off_ok = true; u32 ref_id = args[i].type, type_size; - int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -12958,7 +12923,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(kf_arg_type)) { + !type_may_be_null(arg_type)) { const char *expected_type; expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); @@ -12988,7 +12953,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { ret = mark_arg_precision(env, argno); if (ret) return ret; @@ -13001,54 +12966,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - case KF_ARG_CONST_MEM_SIZE: - case KF_ARG_MEM_SIZE: - case KF_ARG_ANYTHING: - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - case KF_ARG_PTR_TO_BTF_ID: - case KF_ARG_CONST_MAP_PTR: - case KF_ARG_PTR_TO_ITER: - case KF_ARG_PTR_TO_LIST_HEAD: - case KF_ARG_PTR_TO_LIST_NODE: - case KF_ARG_PTR_TO_RB_ROOT: - case KF_ARG_PTR_TO_RB_NODE: - case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_CONST_STR: - case KF_ARG_PTR_TO_WORKQUEUE: - case KF_ARG_PTR_TO_TIMER: - case KF_ARG_PTR_TO_TASK_WORK: - case KF_ARG_PTR_TO_IRQ_FLAG: - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - case KF_ARG_PTR_TO_ARENA: - break; - case KF_ARG_PTR_TO_DYNPTR: - arg_type = ARG_PTR_TO_DYNPTR; - break; - case KF_ARG_PTR_TO_CTX: - arg_type = ARG_PTR_TO_CTX; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - arg_type = ARG_PTR_TO_BTF_ID; - btf_id_fixed_off_ok = false; - break; - default: - verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); - return -EFAULT; - } - if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; - ret = __check_func_arg_reg_off(env, reg, argno, arg_type, - btf_id_fixed_off_ok); + ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) return ret; - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: + switch (base_type(arg_type)) { + case ARG_CONST_SCALAR: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, @@ -13069,7 +12994,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; } break; - case KF_ARG_ANYTHING: + case ARG_SCALAR: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, @@ -13080,7 +13005,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return -EINVAL; } break; - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, @@ -13103,7 +13028,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; } break; - case KF_ARG_PTR_TO_CTX: + case ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); @@ -13122,14 +13047,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me meta->ret_btf_id = ret; } break; - case KF_ARG_PTR_TO_ARENA: + case ARG_PTR_TO_ARENA: if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { verbose(env, "%s is not a pointer to arena or scalar\n", reg_arg_name(env, argno)); return -EINVAL; } break; - case KF_ARG_PTR_TO_ALLOC_BTF_ID: + case ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { verbose(env, "%s expected for bpf_obj_drop()\n", @@ -13165,7 +13090,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me meta->arg_btf_id = reg->btf_id; } break; - case KF_ARG_PTR_TO_DYNPTR: + case ARG_PTR_TO_DYNPTR: { enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; @@ -13200,7 +13125,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; break; } - case KF_ARG_PTR_TO_ITER: + case ARG_PTR_TO_ITER: if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new])) { if (!check_css_task_iter_allowlist(env)) { verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); @@ -13211,7 +13136,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_LIST_HEAD: + case ARG_PTR_TO_LIST_HEAD: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to map value or allocated object\n", @@ -13227,7 +13152,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_RB_ROOT: + case ARG_PTR_TO_RB_ROOT: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to map value or allocated object\n", @@ -13243,7 +13168,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_LIST_NODE: + case ARG_PTR_TO_LIST_NODE: if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { /* Allow bpf_list_front/back return value for @@ -13265,7 +13190,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_RB_NODE: + case ARG_PTR_TO_RB_NODE: if (is_bpf_rbtree_add_kfunc(meta->func_id)) { if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to allocated object\n", @@ -13292,7 +13217,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_CONST_MAP_PTR: + case ARG_CONST_MAP_PTR: if (base_type(reg->type) != CONST_PTR_TO_MAP || type_may_be_null(reg->type)) { verbose(env, "pointer in %s isn't map pointer\n", @@ -13303,7 +13228,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_BTF_ID: + case ARG_PTR_TO_BTF_ID: /* Only base_type is checked, further checks are done here */ if (base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)]) { @@ -13368,10 +13293,10 @@ check_ok: * If the register does not contain btf id but the argument type is a pointer to * scalar-only struct, allow verifying it as a fixed size memory. */ - kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; fallthrough; - case KF_ARG_PTR_TO_MEM: - if (kf_arg_type & MEM_FIXED_SIZE) { + case ARG_PTR_TO_MEM: + if (arg_type & MEM_FIXED_SIZE) { bool known_memory; resolve_ret = btf_resolve_size(btf, ref_t, &type_size); @@ -13405,7 +13330,7 @@ check_ok: } } break; - case KF_ARG_CONST_MEM_SIZE: + case ARG_CONST_MEM_SIZE: ret = process_const_arg(env, reg, argno, meta); if (ret < 0) { if (ret == -EINVAL) @@ -13416,7 +13341,7 @@ check_ok: return ret; } fallthrough; - case KF_ARG_MEM_SIZE: + case ARG_MEM_SIZE: { struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); struct bpf_reg_state *size_reg = reg; @@ -13469,14 +13394,14 @@ check_ok: } break; } - case KF_ARG_PTR_TO_CALLBACK: + case ARG_PTR_TO_FUNC: if (reg->type != PTR_TO_FUNC) { verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); return -EINVAL; } meta->subprogno = reg->subprogno; break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: + case ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); @@ -13505,7 +13430,7 @@ check_ok: meta->arg_btf = reg->btf; meta->arg_btf_id = reg->btf_id; break; - case KF_ARG_PTR_TO_CONST_STR: + case ARG_PTR_TO_CONST_STR: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); @@ -13520,7 +13445,7 @@ check_ok: if (ret) return ret; break; - case KF_ARG_PTR_TO_WORKQUEUE: + case ARG_PTR_TO_WORKQUEUE: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); @@ -13530,7 +13455,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_TIMER: + case ARG_PTR_TO_TIMER: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); @@ -13540,7 +13465,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_TASK_WORK: + case ARG_PTR_TO_TASK_WORK: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); @@ -13550,7 +13475,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_IRQ_FLAG: + case ARG_PTR_TO_IRQ_FLAG: if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); @@ -13565,7 +13490,7 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_RES_SPIN_LOCK: + case ARG_PTR_TO_RES_SPIN_LOCK: { int flags = PROCESS_RES_LOCK; From 5fa6e69e2e6618bfa1061c8012aacf81798e2b1e Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:03:59 -0700 Subject: [PATCH 104/660] bpf: Classify kfunc arguments the verifier ignores The verifier does not inspect __ign arguments or implicit arguments, except that bpf_prog_aux is handled specially. get_kfunc_arg_type() leaves the ignored arguments unclassified, so gen_kfunc_arg_proto() skips them and check_kfunc_args() repeats the same BTF predicates for every call. Add ARG_IGNORE and classify ignored arguments in get_kfunc_arg_type(). The generated prototype can then record them, and the call site can consume the recorded type without deriving the classification again. Keep bpf_prog_aux on its existing BTF-based path for now. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-8-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 1 + kernel/bpf/verifier.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ebfdf9d209f3..54af5562ec95 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -923,6 +923,7 @@ enum bpf_arg_type { ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dc849e670876..09e96d04801d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8838,6 +8838,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 key_size; int err = 0; + if (arg_type == ARG_IGNORE) + return 0; + if (regno >= 0) { err = check_reg_arg(env, regno, SRC_OP); if (err) @@ -12140,6 +12143,9 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const char *ref_tname = NULL; int arg_type; + if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) + return ARG_IGNORE; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); /* Scalar arguments are classified from their BTF suffix/name alone. */ @@ -12269,7 +12275,6 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, struct bpf_func_proto *proto) { - const struct btf *btf = meta->btf; const struct btf_param *args; u32 i, nargs; int arg_type; @@ -12288,9 +12293,7 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg } for (i = 0; i < nargs; i++) { - if (is_kfunc_arg_prog_aux(btf, &args[i]) || - is_kfunc_arg_ignore(btf, &args[i]) || - is_kfunc_arg_implicit(meta, i)) + if (is_kfunc_arg_prog_aux(meta->btf, &args[i])) continue; arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); @@ -12911,7 +12914,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me continue; } - if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) + if (arg_type == ARG_IGNORE) continue; t = btf_type_skip_modifiers(btf, args[i].type, NULL); From 0cc91f0e72415cbd17697e2f8fd0cd8de5c7aae9 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:00 -0700 Subject: [PATCH 105/660] bpf: Align helper and kfunc ARG_PTR_TO_PROG_AUX handling The verifier supplies a bpf_prog_aux argument to both bpf_timer_set_callback() and kfuncs rather than reading it from the BPF program. The helper prototype leaves its third argument unused, while the kfunc path identifies the argument from BTF at every call. Add ARG_PTR_TO_PROG_AUX, record it in the helper prototype, and classify the kfunc argument when its prototype is generated. Because other ignored and implicit arguments are already represented by ARG_IGNORE, the generated prototype has no internal ARG_UNUSED gaps. Validate in one place that it contains at most one prog-aux argument and that the argument is register-passed, as required by the BPF_LD_IMM64 fixup. Record the argument register in the per-instruction metadata for both call kinds. Drive the helper fixup from that metadata instead of the helper ID and its hard-coded R3, matching the existing kfunc fixup. bpf_call_arg_meta::arg_prog is then no longer needed. No functional change beyond reporting invalid kfunc signatures when the call is added rather than when it is verified. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-9-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 1 + include/linux/bpf_verifier.h | 1 - kernel/bpf/fixups.c | 5 +-- kernel/bpf/helpers.c | 1 + kernel/bpf/verifier.c | 66 +++++++++++++++++++++++++----------- 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54af5562ec95..aa4d3bb5e8cc 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -923,6 +923,7 @@ enum bpf_arg_type { ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6b973b94ee75..1e7593e8d5c5 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1599,7 +1599,6 @@ struct bpf_call_arg_meta { struct btf *arg_btf; u32 arg_btf_id; bool arg_owning_ref; - bool arg_prog; struct { struct btf_field *field; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index fcf68cfb91e9..2add8001c3ec 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2020,7 +2020,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (insn->imm == BPF_FUNC_timer_set_callback) { + aux = &env->insn_aux_data[i + delta]; + if (aux->arg_prog) { /* The verifier will process callback_fn as many times as necessary * with different maps and the register states prepared by * set_timer_callback_state will be accurate. @@ -2035,7 +2036,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * bpf_timer_set_callback-ed will return -EINVAL. */ struct bpf_insn ld_addrs[2] = { - BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), + BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), }; insn_buf[0] = ld_addrs[0]; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..051b6654e57c 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1510,6 +1510,7 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_TIMER, .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_PROG_AUX, }; static bool defer_timer_wq_op(void) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 09e96d04801d..c7822addfc64 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8783,6 +8783,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } static bool can_elide_value_nullness(const struct bpf_map *map); +static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_call_arg_meta *meta) @@ -8838,6 +8839,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 key_size; int err = 0; + if (arg_type == ARG_PTR_TO_PROG_AUX) { + cur_aux(env)->arg_prog = regno; + return 0; + } + if (arg_type == ARG_IGNORE) return 0; @@ -9454,9 +9460,42 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ return true; } -static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +static bool check_arg_prog_aux(struct bpf_verifier_env *env, + const struct bpf_func_proto *proto) { - return check_raw_mode_ok(fn, meta) && + bool seen = false; + argno_t argno; + u32 i; + + for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) { + if (proto->arg_type[i] == ARG_UNUSED) + break; + if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX) + continue; + + if (seen) { + verifier_bug(env, "Only 1 prog->aux argument supported"); + return false; + } + + argno = argno_from_arg(i + 1); + if (reg_from_argno(argno) < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", + reg_arg_name(env, argno)); + return false; + } + + seen = true; + } + + return true; +} + +static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn, + struct bpf_call_arg_meta *meta) +{ + return check_arg_prog_aux(env, fn) && + check_raw_mode_ok(fn, meta) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -10999,7 +11038,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); - err = check_func_proto(fn, &meta); + err = check_func_proto(env, fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; @@ -12143,6 +12182,9 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const char *ref_tname = NULL; int arg_type; + if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) + return ARG_PTR_TO_PROG_AUX; + if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) return ARG_IGNORE; @@ -12293,9 +12335,6 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg } for (i = 0; i < nargs; i++) { - if (is_kfunc_arg_prog_aux(meta->btf, &args[i])) - continue; - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); if (arg_type < 0) return arg_type; @@ -12303,7 +12342,7 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg proto->arg_type[i] = arg_type; } - return 0; + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12898,18 +12937,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me int regno = reg_from_argno(argno); u32 ref_id = args[i].type, type_size; - if (is_kfunc_arg_prog_aux(btf, &args[i])) { - /* Reject repeated use bpf_prog_aux */ - if (meta->arg_prog) { - verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); - return -EFAULT; - } - if (regno < 0) { - verbose(env, "%s prog->aux cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->arg_prog = true; + if (arg_type == ARG_PTR_TO_PROG_AUX) { cur_aux(env)->arg_prog = regno; continue; } From 57e181cde7e6bfbb90fcd60d1e96153f529ffe2e Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:01 -0700 Subject: [PATCH 106/660] bpf: Set OBJ_RELEASE when generating kfunc argument types check_kfunc_args() ORs OBJ_RELEASE into arg_type on every verification of every argument of every call, from a comparison against meta->release_regno. Neither side of that comparison depends on the call site. bpf_fetch_kfunc_arg_meta() sets release_regno to BPF_REG_1 when the kfunc is KF_RELEASE and leaves it zero otherwise, and regno is derived from the argument index, so the test is true exactly for argument 0 of a KF_RELEASE kfunc. Set the flag in get_kfunc_arg_type() instead, where the rest of the classification is built, and drop the comparison. The release handling in check_func_proto() is helper-only, so an OBJ_RELEASE in a generated kfunc prototype does not feed back into meta->release_regno. meta->release_regno itself stays: it is still what tells the referenced PTR_TO_BTF_ID check and release_reg() which register to act on. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-10-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c7822addfc64..20e7da6674a1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12311,6 +12311,13 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, if (is_kfunc_arg_nullable(meta->btf, &args[arg])) arg_type |= PTR_MAYBE_NULL; + /* + * Only the first argument of a KF_RELEASE kfunc releases anything, and + * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it. + */ + if (is_kfunc_release(meta) && arg == 0) + arg_type |= OBJ_RELEASE; + return arg_type; } @@ -12997,8 +13004,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - if (regno == meta->release_regno) - arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) return ret; From 78811eac7ae6e4510827f320d780b3c417601142 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:02 -0700 Subject: [PATCH 107/660] bpf: Set MEM_UNINIT and dynptr subtypes when generating kfunc arg types The dynptr flavour a kfunc argument expects is rebuilt on every verification of the call: check_kfunc_args() starts from ARG_PTR_TO_DYNPTR, adds MEM_UNINIT from the __uninit suffix, and then walks a chain of func_id comparisons to add DYNPTR_TYPE_SKB, _XDP, _SKB_META or _FILE. All of that is fixed by the kfunc identity and BTF, so move it into get_kfunc_arg_type() and let the generated prototype carry it. Inheriting the classification also means PTR_MAYBE_NULL reaches process_dynptr_func() for a __nullable dynptr argument, where it did not before, and is_dynptr_type_expected() tested for any type of dynptr with an exact arg_type == ARG_PTR_TO_DYNPTR comparison. Test the DYNPTR_TYPE_FLAG_MASK bits instead, which is what the comment there already claims to mean and does not care about unrelated flags. What cannot move is the KF_bpf_dynptr_clone arm, which takes its type from meta->dynptr.type, recorded while verifying the parent dynptr argument earlier in the same call. That stays at the call site, applied on top of the cached classification. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-11-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 45 ++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 20e7da6674a1..c602007bf583 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -960,8 +960,12 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { - /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ - if (arg_type == ARG_PTR_TO_DYNPTR) + /* + * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr. + * Test the flags rather than the whole arg_type, which may carry + * unrelated ones such as PTR_MAYBE_NULL. + */ + if (!(arg_type & DYNPTR_TYPE_FLAG_MASK)) return true; return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); @@ -12227,9 +12231,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, arg_type = ARG_PTR_TO_ALLOC_BTF_ID; else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; - else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) { arg_type = ARG_PTR_TO_DYNPTR; - else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) + arg_type |= DYNPTR_TYPE_SKB; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) + arg_type |= DYNPTR_TYPE_XDP; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) + arg_type |= DYNPTR_TYPE_SKB_META; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) + /* OBJ_RELEASE for the latter comes from KF_RELEASE below */ + arg_type |= DYNPTR_TYPE_FILE; + } else if (is_kfunc_arg_iter(meta, arg, &args[arg])) arg_type = ARG_PTR_TO_ITER; else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) arg_type = ARG_PTR_TO_LIST_HEAD; @@ -12308,6 +12323,9 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } + if (is_kfunc_arg_uninit(meta->btf, &args[arg])) + arg_type |= MEM_UNINIT; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) arg_type |= PTR_MAYBE_NULL; @@ -13128,23 +13146,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; case ARG_PTR_TO_DYNPTR: { - enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; + enum bpf_arg_type dynptr_arg_type = arg_type; - if (is_kfunc_arg_uninit(btf, &args[i])) - dynptr_arg_type |= MEM_UNINIT; - - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) { - dynptr_arg_type |= DYNPTR_TYPE_SKB; - } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) { - dynptr_arg_type |= DYNPTR_TYPE_XDP; - } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) { - dynptr_arg_type |= DYNPTR_TYPE_SKB_META; - } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file])) { - dynptr_arg_type |= DYNPTR_TYPE_FILE; - } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) { - dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]) && - (dynptr_arg_type & MEM_UNINIT)) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]) && + (dynptr_arg_type & MEM_UNINIT)) { enum bpf_dynptr_type parent_type = meta->dynptr.type; if (parent_type == BPF_DYNPTR_TYPE_INVALID) { From 2b2b08c277cb2ba2596e88d6a2efdb9df3117b75 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:03 -0700 Subject: [PATCH 108/660] bpf: Set MEM_RCU when generating kfunc argument types The BTF ID argument of a kfunc has to be referenced or trusted unless the kfunc is KF_RCU, in which case an RCU-protected pointer is accepted too. check_kfunc_args() reads that off meta->kfunc_flags on every verification of the call, which is the last thing keeping the argument policy of ARG_PTR_TO_BTF_ID out of the generated prototype. Set MEM_RCU in get_kfunc_arg_type() for ARG_PTR_TO_BTF_ID arguments of KF_RCU kfuncs, the way it already sets OBJ_RELEASE for release arguments, and test the generated flag instead. MEM_RCU represents the KF_RCU relaxation applied by the ARG_PTR_TO_BTF_ID trust check; other argument kinds have their own provenance requirements. No functional change intended. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-12-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c602007bf583..39f7f633b4d6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12336,6 +12336,14 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, if (is_kfunc_release(meta) && arg == 0) arg_type |= OBJ_RELEASE; + /* + * A KF_RCU kfunc accepts an RCU-protected pointer where it would + * otherwise demand a referenced or trusted one. Other argument kinds + * have their own provenance requirements and must not inherit MEM_RCU. + */ + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID && is_kfunc_rcu(meta)) + arg_type |= MEM_RCU; + return arg_type; } @@ -13275,7 +13283,7 @@ check_ok: reg2btf_ids[base_type(reg->type)]) { if (!is_trusted_reg(env, reg) || bpf_type_has_unsafe_modifiers(reg->type)) { - if (!is_kfunc_rcu(meta)) { + if (!(arg_type & MEM_RCU)) { const char *expected_type; expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); From fb8d353bb58f4928337591ef2c49820fb71faff2 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:04 -0700 Subject: [PATCH 109/660] bpf: Resolve BTF ID of ARG_PTR_TO_BTF_ID in kfunc bpf_func_proto check_kfunc_args() walks the kfunc's BTF on every verification of a call to work out which BTF ID an ARG_PTR_TO_BTF_ID argument expects. Helpers name theirs in bpf_func_proto::arg_btf_id[], as a pointer to a BTF ID that resolve_btfids fills in at build time. The ID of a kfunc argument's referent is already stored in the immutable BTF records that describe its pointer and modifier chain. Make arg_btf_id[] point to the BTF field containing the resolved ID. A __map argument instead uses the existing vmlinux BTF ID pointer. Produce this metadata alongside the argument classification in get_kfunc_arg_type(). Declare the argument BTF ID pointers const, since the verifier only reads through them. The BTF object owns their storage and remains alive while the generated prototype is used, so the pointers remain valid when the kfunc descriptor array is reallocated or sorted. Both helper and kfunc callers can now read the expected BTF ID through the same bpf_func_proto field, which lets check_func_arg() take over the ARG_PTR_TO_BTF_ID case. arg_btf_id[] shares a union with arg_size[], so an argument cannot store both. The scalar-struct memory fallback keeps resolving its byte size at verification time, as it does today. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-13-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 12 ++++++------ kernel/bpf/verifier.c | 42 ++++++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index aa4d3bb5e8cc..d0066d744ceb 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1021,13 +1021,13 @@ struct bpf_func_proto { }; union { struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; + const u32 *arg1_btf_id; + const u32 *arg2_btf_id; + const u32 *arg3_btf_id; + const u32 *arg4_btf_id; + const u32 *arg5_btf_id; }; - u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; + const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 39f7f633b4d6..0547fbeeebbe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8839,7 +8839,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, enum bpf_arg_type arg_type = fn->arg_type[arg]; int regno = reg_from_argno(argno); enum bpf_reg_type type = reg->type; - u32 *arg_btf_id = NULL; + const u32 *arg_btf_id = NULL; u32 key_size; int err = 0; @@ -12179,13 +12179,18 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs) + const struct btf_param *args, int arg, int nargs, + struct bpf_func_proto *proto) { const struct btf_type *t, *ref_t = NULL; + const u32 *ref_id_ptr = NULL; argno_t argno = argno_from_arg(arg + 1); const char *ref_tname = NULL; + u32 ref_id; int arg_type; + proto->arg_btf_id[arg] = NULL; + if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) return ARG_PTR_TO_PROG_AUX; @@ -12213,7 +12218,11 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } - ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + /* Keep a pointer to the BTF field containing the resolved referent ID. */ + ref_id_ptr = &t->type; + ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id); + while (*ref_id_ptr != ref_id) + ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type; ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, @@ -12336,13 +12345,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, if (is_kfunc_release(meta) && arg == 0) arg_type |= OBJ_RELEASE; - /* - * A KF_RCU kfunc accepts an RCU-protected pointer where it would - * otherwise demand a referenced or trusted one. Other argument kinds - * have their own provenance requirements and must not inherit MEM_RCU. - */ - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID && is_kfunc_rcu(meta)) - arg_type |= MEM_RCU; + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + if (is_kfunc_arg_map(meta->btf, &args[arg])) + proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP]; + else + proto->arg_btf_id[arg] = ref_id_ptr; + + /* + * A KF_RCU kfunc accepts an RCU-protected pointer where it would + * otherwise demand a referenced or trusted one. Other argument kinds + * have their own provenance requirements and must not inherit MEM_RCU. + */ + if (is_kfunc_rcu(meta)) + arg_type |= MEM_RCU; + } return arg_type; } @@ -12368,7 +12384,7 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg } for (i = 0; i < nargs; i++) { - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, proto); if (arg_type < 0) return arg_type; @@ -13024,8 +13040,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me continue; } + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) + ref_id = *meta->fn->arg_btf_id[i]; + if (is_kfunc_arg_map(btf, &args[i])) { - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; ref_t = btf_type_by_id(btf_vmlinux, ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); } From 6ca151e7cc320c68861c411c72dd0178c6fcde62 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:05 -0700 Subject: [PATCH 110/660] bpf: Resolve ARG_PTR_TO_MEM | MEM_FIXED_SIZE size in kfunc bpf_func_proto A generated kfunc prototype classifies a pointer to scalar memory without an adjacent size argument as ARG_PTR_TO_MEM | MEM_FIXED_SIZE. The kfunc path currently walks BTF and resolves the pointee size each time it verifies a call. The common check_func_arg() path instead expects that size in bpf_func_proto::arg_size[]. Resolve the size when generating the prototype and teach the existing kfunc path to consume the cached value. This prepares fixed-size memory arguments to move to the common checker. arg_size[] shares storage with arg_btf_id[]. An ARG_PTR_TO_BTF_ID argument that falls back to scalar-struct memory therefore keeps its BTF ID pointer and continues resolving the size when that fallback is taken. No functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-14-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0547fbeeebbe..615a5667e569 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12182,11 +12182,11 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const struct btf_param *args, int arg, int nargs, struct bpf_func_proto *proto) { - const struct btf_type *t, *ref_t = NULL; + const struct btf_type *t, *ref_t = NULL, *resolve_ret; const u32 *ref_id_ptr = NULL; argno_t argno = argno_from_arg(arg + 1); const char *ref_tname = NULL; - u32 ref_id; + u32 ref_id, type_size; int arg_type; proto->arg_btf_id[arg] = NULL; @@ -12329,6 +12329,15 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } @@ -12984,7 +12993,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me const struct btf_type *t, *ref_t, *resolve_ret; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); - u32 ref_id = args[i].type, type_size; + u32 ref_id = args[i].type; + u32 arg_size = arg_type & MEM_FIXED_SIZE ? meta->fn->arg_size[i] : 0; if (arg_type == ARG_PTR_TO_PROG_AUX) { cur_aux(env)->arg_prog = regno; @@ -13360,20 +13370,21 @@ check_ok: * If the register does not contain btf id but the argument type is a pointer to * scalar-only struct, allow verifying it as a fixed size memory. */ + resolve_ret = btf_resolve_size(btf, ref_t, &arg_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; fallthrough; case ARG_PTR_TO_MEM: if (arg_type & MEM_FIXED_SIZE) { bool known_memory; - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, + ret = check_mem_reg(env, reg, argno, arg_size, BPF_READ | BPF_WRITE, meta, &known_memory); if (ret < 0) { const char *expected_type; @@ -13384,14 +13395,14 @@ check_ok: env, insn_idx, argno, func_name, "Pass memory with at least the required number of accessible bytes and suitable read and write access.", "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", - type_size, expected_type, + arg_size, expected_type, bpf_diag_reg_type_plain(env, reg->type)); else bpf_diag_call_arg_fmt( env, insn_idx, argno, func_name, "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", - type_size, expected_type, + arg_size, expected_type, bpf_diag_reg_type_plain(env, reg->type)); return ret; } From 2e418779a00e0f95bcfe3177c71ea196e0134765 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:06 -0700 Subject: [PATCH 111/660] bpf: Consolidate runtime argument type resolution A function argument type can require call-site refinement before register admission. Helpers refine map-value arguments according to the recorded map type, while kfuncs reinterpret a pointer to a scalar-only BTF struct as fixed-size memory when the register does not carry a BTF ID. Add resolve_func_arg_type() as the common entry point. It delegates helper map-value refinement to the existing resolve_map_arg_type(), then handles the kfunc BTF-to-memory fallback. It updates the effective argument type and size consumed by the subsequent checks. Preserve PTR_MAYBE_NULL when changing the base type so later nullability validation observes the BTF contract. Keep OBJ_RELEASE arguments in their original pointer class. The latter is not a bug fix: existing kfunc checking already validates ownership before this resolver. It preserves release metadata for the later shared checking order, where ownership validation follows register admission. Moving the kfunc fallback before admission lets the resulting fixed-size memory argument enter the compatibility check directly. This is another step toward routing both call types through check_func_arg(). Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-15-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 192 ++++++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 84 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 615a5667e569..4e7230cb5791 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8196,8 +8196,8 @@ static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) } static int resolve_map_arg_type(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, - enum bpf_arg_type *arg_type) + const struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type) { if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ @@ -8225,6 +8225,11 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, return 0; } +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, + struct bpf_call_arg_meta *meta, int insn_idx, + enum bpf_arg_type *arg_type, u32 *arg_size); + struct bpf_reg_types { const enum bpf_reg_type types[10]; u32 *btf_id; @@ -8840,6 +8845,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, int regno = reg_from_argno(argno); enum bpf_reg_type type = reg->type; const u32 *arg_btf_id = NULL; + u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; @@ -8872,11 +8878,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EACCES; } - if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { - err = resolve_map_arg_type(env, meta, &arg_type); - if (err) - return err; - } + err = resolve_func_arg_type(env, reg, arg, meta, insn_idx, &arg_type, &arg_size); + if (err) + return err; if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { /* A NULL register has a SCALAR_VALUE type, so skip @@ -9017,12 +9021,12 @@ skip_type_check: * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno, fn->arg_size[arg], + err = check_mem_reg(env, reg, argno, arg_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; if (arg_type & MEM_ALIGNED) - err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); + err = check_ptr_alignment(env, reg, 0, arg_size, true); } break; case ARG_MEM_SIZE: @@ -11911,6 +11915,65 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, + struct bpf_call_arg_meta *meta, int insn_idx, + enum bpf_arg_type *arg_type, u32 *arg_size) +{ + argno_t argno = argno_from_arg(arg + 1); + const struct btf_param *args; + const struct btf_type *ref_t, *resolve_ret; + const struct btf *btf; + const char *ref_tname; + u32 ref_id; + + if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE) + return resolve_map_arg_type(env, meta, arg_type); + + if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) + return 0; + + if (!meta->btf || arg_type_is_release(*arg_type) || + base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) + return 0; + args = btf_params(meta->func_proto); + ref_id = *meta->fn->arg_btf_id[arg]; + btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; + ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + + if (!btf_type_is_scalar_struct(env, btf, ref_t)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + const char *expected_type; + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", + "the function expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", + expected_type, + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + + resolve_ret = btf_resolve_size(btf, ref_t, arg_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, + PTR_ERR(resolve_ret)); + return -EINVAL; + } + *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | (*arg_type & PTR_MAYBE_NULL); + + return 0; +} + static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, char *buf, size_t buf_sz) { @@ -12990,7 +13053,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me for (i = 0; i < nargs; i++) { struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); enum bpf_arg_type arg_type = meta->fn->arg_type[i]; - const struct btf_type *t, *ref_t, *resolve_ret; + const struct btf_type *t, *ref_t; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); u32 ref_id = args[i].type; @@ -13053,11 +13116,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) ref_id = *meta->fn->arg_btf_id[i]; - if (is_kfunc_arg_map(btf, &args[i])) { - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - + ret = resolve_func_arg_type(env, reg, i, meta, insn_idx, &arg_type, &arg_size); + if (ret < 0) + return ret; ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) return ret; @@ -13306,80 +13367,43 @@ check_ok: return ret; break; case ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if (base_type(reg->type) == PTR_TO_BTF_ID || - reg2btf_ids[base_type(reg->type)]) { - if (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type)) { - if (!(arg_type & MEM_RCU)) { - const char *expected_type; + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!(arg_type & MEM_RCU)) { + const char *actual_type, *arg_name, *expected_type; - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", - "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Use this kfunc with a pointer that is valid in an RCU read lock region.", - "the kfunc requires an RCU-protected pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", + "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; } + if (!is_rcu_reg(reg)) { + const char *actual_type, *arg_name, *expected_type; - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Use this kfunc with a pointer that is valid in an RCU read lock region.", + "the kfunc requires an RCU-protected pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { - enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); - const char *expected_type; - - verbose(env, "%s is %s expected %s %s", - reg_arg_name(env, argno), reg_type_str(env, reg->type), - btf_type_str(ref_t), ref_tname); - if (reg2btf_type != NOT_INIT) - verbose(env, " or %s", reg_type_str(env, reg2btf_type)); - verbose(env, "\n"); - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", - "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", - expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - /* - * If the register does not contain btf id but the argument type is a pointer to - * scalar-only struct, allow verifying it as a fixed size memory. - */ - resolve_ret = btf_resolve_size(btf, ref_t, &arg_size); - if (IS_ERR(resolve_ret)) { - verbose(env, - "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; - fallthrough; + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, + ref_id, meta, i, argno); + if (ret < 0) + return ret; + break; case ARG_PTR_TO_MEM: if (arg_type & MEM_FIXED_SIZE) { bool known_memory; From 750f3770809ac084814d94424756c9ca8a5ae922 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:07 -0700 Subject: [PATCH 112/660] bpf: Consolidate nullable argument validation check_kfunc_args() rejects a NULL or possibly-NULL register passed to a non-nullable pointer argument up front. Helpers only do so inside check_reg_type() for PTR_TO_BTF_ID arguments, leaving other pointer kinds to fail indirectly through compatible_reg_types[]. Give both paths the same policy and factor it into check_func_arg_nullability(). Gate it on arg_type_is_scalar() rather than on the BTF shape of the kfunc parameter. This keeps a zero passed to a size argument from being interpreted as a NULL pointer. Taking nullability from the argument classification also avoids an is_kfunc_arg_nullable() lookup for every kfunc argument on every verification. Keep release-argument nullability separate from ownership checking. A possibly-NULL release argument must be rejected even if it is otherwise nullable, while the ownership check must still permit a literal NULL for bpf_kptr_xchg(). Use call-neutral structured diagnostics and retain the expected BTF type when it is available for a kfunc argument. Resolve that type lazily from the call metadata only when reporting an error. Helpers now report NULL-ness rather than a register-type mismatch for non-BTF-ID pointer arguments and literal NULL passed to a non-nullable pointer argument. Update the affected selftest expectations. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-16-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 110 ++++++++++++------ .../selftests/bpf/prog_tests/cb_refs.c | 2 +- .../selftests/bpf/progs/cpumask_failure.c | 2 +- .../selftests/bpf/progs/iters_testmod.c | 4 +- .../selftests/bpf/progs/map_kptr_fail.c | 6 +- .../selftests/bpf/progs/verifier_ctx.c | 2 +- .../progs/verifier_helper_access_var_len.c | 4 +- .../selftests/bpf/progs/verifier_live_stack.c | 2 +- .../selftests/bpf/progs/verifier_map_in_map.c | 3 +- .../bpf/progs/verifier_map_lookup_refine.c | 2 +- .../bpf/progs/verifier_ref_tracking.c | 4 +- .../selftests/bpf/progs/verifier_sock.c | 4 +- .../testing/selftests/bpf/verifier/map_kptr.c | 2 +- 13 files changed, 94 insertions(+), 53 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4e7230cb5791..541467f9101f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8180,6 +8180,25 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } +/* + * An argument that only ever takes a scalar, so a zero register passed to it + * is a value rather than a NULL pointer. + */ +static bool arg_type_is_scalar(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + case ARG_CONST_MEM_SIZE: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return true; + default: + return false; + } +} + /* * A kfunc is named by a BTF ID, which can take the same numeric value as an * enum bpf_func_id. Only test meta->func_id against a BPF_FUNC_* once the call @@ -8382,6 +8401,34 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); } +static int check_func_arg_nullability(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) || + (!bpf_register_is_null(reg) && !type_may_be_null(reg->type))) + return 0; + + if (meta->btf) { + u32 arg_btf_id; + + arg_btf_id = btf_params(meta->func_proto)[arg_idx_from_argno(argno)].type; + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); + } + + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Add a NULL check and make the call only on the non-NULL path.", + "the pointer may be NULL, but this call requires a non-NULL %s", + expected_type); + return -EACCES; +} + static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, const enum bpf_reg_type *types, int count) { @@ -8494,17 +8541,6 @@ found: bool strict_type_match = arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release); - if (type_may_be_null(reg->type) && - (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { - verbose(env, "Possibly NULL pointer passed to helper %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, env->insn_idx, argno, meta->func_name, - "the pointer may be NULL, but this call requires a non-NULL pointer", - "Add a NULL check and make the call only on the non-NULL path."); - return -EACCES; - } - if (!arg_btf_id) { if (!compatible->btf_id) { verifier_bug(env, "missing arg compatible BTF ID"); @@ -8892,6 +8928,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, goto skip_type_check; } + err = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); + if (err) + return err; + /* arg_btf_id and arg_size are in a union. */ if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) @@ -8906,15 +8946,28 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; skip_type_check: - if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - meta->func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, meta->func_name, - "release helpers require a value that owns a live resource returned by a matching acquire helper", - "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); - return -EINVAL; + if (arg_type_is_release(arg_type)) { + if (type_may_be_null(reg->type)) { + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "the pointer may be NULL, but this call requires a non-NULL pointer", + "Add a NULL check and make the call only on the non-NULL path."); + return -EACCES; + } + + if (!arg_type_is_dynptr(arg_type) && + !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + verbose(env, + "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", + meta->func_name, reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "release helpers require a value that owns a live resource returned by a matching acquire helper", + "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); + return -EINVAL; + } } if (reg_is_referenced(env, reg)) @@ -13074,20 +13127,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - if (btf_type_is_ptr(t) && - (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(arg_type)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Add a NULL check and call the kfunc only on the non-NULL path.", - "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", - expected_type); - return -EACCES; - } + ret = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); + if (ret < 0) + return ret; if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index 78566b817fd7..50ea3d72d446 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -12,7 +12,7 @@ struct { const char *err_msg; } cb_refs_tests[] = { { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, - { "leak_prog", "Possibly NULL pointer passed to helper R2" }, + { "leak_prog", "Possibly NULL pointer passed to trusted R2" }, { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ }; diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 4628feb53d86..6d730535eb95 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -183,7 +183,7 @@ int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to helper R2") +__failure __msg("Possibly NULL pointer passed to trusted R2") int BPF_PROG(test_global_mask_rcu_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *prev, *curr; diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 76012dbbdb41..5a3ff65e8234 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -135,7 +135,7 @@ int iter_ret_rcu_test_protected(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type(const void *ctx) { struct task_struct *p; @@ -158,7 +158,7 @@ int iter_ret_rcu_test_protected_nostruct(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rdonly_rcu_mem_or_null expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type_nostruct(const void *ctx) { void *p; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index eee35d203b66..0d7365b704b1 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -149,7 +149,7 @@ int reject_bad_type_match(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=untrusted_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int marked_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -217,7 +217,7 @@ int reject_kptr_xchg_on_unref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mark_ref_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -364,7 +364,7 @@ int kptr_xchg_ref_state(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to helper R2") +__failure __msg("Possibly NULL pointer passed to trusted R2") int kptr_xchg_possibly_null(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; diff --git a/tools/testing/selftests/bpf/progs/verifier_ctx.c b/tools/testing/selftests/bpf/progs/verifier_ctx.c index 7856dad3d1f3..9d42ba824408 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ctx.c @@ -208,7 +208,7 @@ __naked void null_check_7_ctx_bind(void) SEC("cgroup/post_bind4") __description("pass ctx or null check, 8: null (bind)") -__failure __msg("R1 type=scalar expected=ctx") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void null_check_8_null_bind(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index 343fc08d9747..d1452ef6f2f9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -621,7 +621,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_8(void) { asm volatile (" \ @@ -637,7 +637,7 @@ __naked void ptr_to_mem_or_null_8(void) SEC("tracepoint") __description("helper access to variable memory: size > 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_9(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 401152b2b64f..bc3dfdc1a536 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -246,7 +246,7 @@ static __used __naked void read_first_param2(void) SEC("socket") __flag(BPF_F_TEST_STATE_FREQ) __failure -__msg("R1 type=scalar expected=map_ptr") +__msg("Possibly NULL pointer passed to trusted R1") __naked void caller_stack_pruning_callback(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index d3be69a9a755..621248a02a1f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -154,8 +154,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") -__msg("map_ptr_or_null, but this argument accepts map_ptr") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c index c01abf54923d..4b1eadddd89c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -58,7 +58,7 @@ int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) } SEC("?tc") -__failure __msg("type=map_ptr_or_null expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) { char th[sizeof(struct tcphdr)] = {}; diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c index 199ad18f8eb5..f3fad911b5bc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c @@ -344,7 +344,7 @@ __naked void potential_reference_to_system_key(void) SEC("tc") __description("reference tracking: release reference without check") -__failure __msg("type=sock_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void tracking_release_reference_without_check(void) { asm volatile ( @@ -363,7 +363,7 @@ __naked void tracking_release_reference_without_check(void) SEC("tc") __description("reference tracking: release reference to sock_common without check") -__failure __msg("type=sock_common_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void to_sock_common_without_check(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index 4f2f3209eec8..d59b2f905671 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -110,7 +110,7 @@ l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_type]); \ SEC("cgroup/skb") __description("bpf_sk_fullsock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_1(void) { @@ -466,7 +466,7 @@ l1_%=: r0 = *(u32*)(r0 + %[bpf_sock_rx_queue_mapping__end]);\ SEC("cgroup/skb") __description("bpf_tcp_sock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_2(void) { diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index 4b39f8472f9b..1efaff296b7c 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c @@ -311,7 +311,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "R1 type=rcu_ptr_or_null_ expected=percpu_ptr_", + .errstr = "Possibly NULL pointer passed to trusted R1", }, { "map_kptr: ref: reject off != 0", From a583d56e8de3e5300108594fe11d21e11df52a0d Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:08 -0700 Subject: [PATCH 113/660] bpf: Drop redundant BTF pointer helper write rejection check_reg_type() rejects a trusted PTR_TO_BTF_ID admitted by mem_types unless the argument carries MEM_RDONLY. This duplicates argument-specific memory access validation and is too conservative for map keys and input map values, whose read direction is not represented by MEM_RDONLY. Since commit fa75b7c85b0d ("bpf: Enforce write checks for BTF pointer helper access"), check_helper_mem_access() forwards the requested access type to check_ptr_to_btf_access(). Map keys always request read access. Map values request write access only when tagged with MEM_WRITE. ARG_PTR_TO_MEM prototypes must carry either MEM_RDONLY or MEM_WRITE, and size arguments use the direction of their associated memory argument. check_ptr_to_btf_access() therefore checks the effective access direction in every case. Drop the early rejection and keep check_reg_type() limited to register admission. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-17-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 10 +--------- tools/testing/selftests/bpf/progs/task_kfunc_failure.c | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 541467f9101f..eb825fc3ce0d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8515,16 +8515,8 @@ found: if (base_type(reg->type) != PTR_TO_BTF_ID) return 0; - if (compatible == &mem_types) { - if (!(arg_type & MEM_RDONLY)) { - verbose(env, - "%s() may write into memory pointed by %s type=%s\n", - meta->func_name, - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - return -EACCES; - } + if (compatible == &mem_types) return 0; - } switch ((int)reg->type) { case PTR_TO_BTF_ID: diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index f96b0c13ed1a..9979766d4d74 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -333,7 +333,7 @@ int BPF_PROG(task_access_comm2, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("write into memory") +__failure __msg("only read is supported") int BPF_PROG(task_access_comm3, struct task_struct *task, u64 clone_flags) { bpf_probe_read_kernel(task->comm, 16, task->comm); From b472f03e3141319f44c59b047f5667b29fc5f66e Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:09 -0700 Subject: [PATCH 114/660] bpf: Consolidate helper and kfunc PTR_TO_BTF_ID argument matching Keep check_reg_type() focused on register admission. Helpers currently match BTF-ID arguments there, while kfuncs use process_kf_arg_ptr_to_btf_id(). Both paths ultimately call btf_struct_ids_match(). Introduce process_arg_ptr_to_btf_id() for helpers, kfuncs, and global subprograms. Callers provide the expected BTF and ID and retain their call-specific metadata handling. Group the helper compatible-ID, poison, and bpf_kptr_xchg() handling in a helper-only block in the ARG_PTR_TO_BTF_ID case, leaving the common matcher outside it. Derive strict matching from the generated argument type. This limits KF_RELEASE strictness to the argument marked OBJ_RELEASE while preserving the bpf_sk_release() exception and kfunc no-cast-alias rule. Remove the post-admission BTF and nullability switch from check_reg_type(), leaving it responsible for register admission. The compatibility tables already limit helper MEM_ALLOC inputs to ARG_PTR_TO_SPIN_LOCK and ARG_KPTR_XCHG_DEST, while the kptr source is admitted only for bpf_kptr_xchg(). Drop the redundant helper-ID whitelist, pointer-offset check, and constant-offset assertion. The bpf_kptr_xchg() source match now follows offset validation, so a source within a referenced object reports the release zero-offset error before the kptr type error. Update the affected selftests and use call-neutral wording for BTF mismatch diagnostics. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-18-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 215 +++++++----------- .../testing/selftests/bpf/prog_tests/bpf_nf.c | 14 +- .../selftests/bpf/progs/map_kptr_fail.c | 2 +- .../selftests/bpf/progs/verifier_vfs_reject.c | 2 +- tools/testing/selftests/bpf/verifier/calls.c | 2 +- .../testing/selftests/bpf/verifier/map_kptr.c | 2 +- 6 files changed, 94 insertions(+), 143 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index eb825fc3ce0d..11d28c8d9c79 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8248,6 +8248,10 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 arg, struct bpf_call_arg_meta *meta, int insn_idx, enum bpf_arg_type *arg_type, u32 *arg_size); +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx); struct bpf_reg_types { const enum bpf_reg_type types[10]; @@ -8450,13 +8454,12 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, } static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta) + enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; const char *actual, *accepted; - int i, j, err; + int i, j; compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { @@ -8512,90 +8515,6 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re return -EACCES; found: - if (base_type(reg->type) != PTR_TO_BTF_ID) - return 0; - - if (compatible == &mem_types) - return 0; - - switch ((int)reg->type) { - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_TRUSTED: - case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | MEM_RCU: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: - { - /* For bpf_sk_release, it needs to match against first member - * 'struct sock_common', hence make an exception for it. This - * allows bpf_sk_release to work for multiple socket types. - */ - bool strict_type_match = arg_type_is_release(arg_type) && - !is_helper_call(meta, BPF_FUNC_sk_release); - - if (!arg_btf_id) { - if (!compatible->btf_id) { - verifier_bug(env, "missing arg compatible BTF ID"); - return -EFAULT; - } - arg_btf_id = compatible->btf_id; - } - - if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) { - if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) - return -EACCES; - } else { - if (arg_btf_id == BPF_PTR_POISON) { - verbose(env, "verifier internal error:"); - verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", - reg_arg_name(env, argno)); - return -EACCES; - } - - err = __check_ptr_off_reg(env, reg, argno, true); - if (err) - return err; - - if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, - reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match, !type_is_alloc(reg->type))) { - verbose(env, "%s is of type %s but %s is expected\n", - reg_arg_name(env, argno), - btf_type_name(reg->btf, reg->btf_id), - btf_type_name(btf_vmlinux, *arg_btf_id)); - return -EACCES; - } - } - break; - } - case PTR_TO_BTF_ID | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: - if (!is_helper_call(meta, BPF_FUNC_spin_lock) && - !is_helper_call(meta, BPF_FUNC_spin_unlock) && - !is_helper_call(meta, BPF_FUNC_kptr_xchg)) { - verifier_bug(env, "unimplemented handling of MEM_ALLOC"); - return -EFAULT; - } - /* Check if local kptr in src arg matches kptr in dst arg */ - if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) { - int regno = reg_from_argno(argno); - - if (regno == BPF_REG_2 && - map_kptr_match_type(env, meta->kptr_field, reg, regno)) - return -EACCES; - } - break; - case PTR_TO_BTF_ID | MEM_PERCPU: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: - case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: - /* Handled by helper specific checks */ - break; - default: - verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match"); - return -EFAULT; - } return 0; } @@ -8872,7 +8791,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, enum bpf_arg_type arg_type = fn->arg_type[arg]; int regno = reg_from_argno(argno); enum bpf_reg_type type = reg->type; - const u32 *arg_btf_id = NULL; u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; @@ -8924,12 +8842,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; - /* arg_btf_id and arg_size are in a union. */ - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || - base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) - arg_btf_id = fn->arg_btf_id[arg]; - - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, meta); if (err) return err; @@ -9027,6 +8940,43 @@ skip_type_check: arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); break; + case ARG_PTR_TO_BTF_ID: + case ARG_PTR_TO_BTF_ID_SOCK_COMMON: + { + const u32 *arg_btf_id = fn->arg_btf_id[arg]; + + if (!meta->btf) { + const struct bpf_reg_types *compatible; + + if (base_type(reg->type) != PTR_TO_BTF_ID) + break; + + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) + return map_kptr_match_type(env, meta->kptr_field, reg, regno) ? + -EACCES : 0; + + if (!arg_btf_id) { + compatible = compatible_reg_types[base_type(arg_type)]; + if (!compatible->btf_id) { + verifier_bug(env, "missing arg compatible BTF ID"); + return -EFAULT; + } + arg_btf_id = compatible->btf_id; + } + if (arg_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", + reg_arg_name(env, argno)); + return -EACCES; + } + } + + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, btf_vmlinux, + *arg_btf_id, meta, insn_idx); + if (err < 0) + return err; + break; + } case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); @@ -9973,8 +9923,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; } - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, argno, arg->arg_type, &meta); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); + if (!err && base_type(reg->type) == PTR_TO_BTF_ID) + err = process_arg_ptr_to_btf_id(env, reg, argno, arg->arg_type, + btf_vmlinux, arg->btf_id, + &meta, env->insn_idx); if (err) return err; } else { @@ -12511,32 +12465,28 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } -static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, - const struct btf_type *ref_t, - const char *ref_tname, u32 ref_id, - struct bpf_call_arg_meta *meta, - int arg, argno_t argno) +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx) { - const struct btf_type *reg_ref_t; - bool strict_type_match = false; + bool taking_projection, struct_same, strict_type_match = false; + const struct btf_type *arg_t, *reg_t; + const char *arg_tname, *reg_tname; const struct btf *reg_btf; - const char *reg_ref_tname; - bool taking_projection; - bool struct_same; - u32 reg_ref_id; + u32 reg_btf_id; if (base_type(reg->type) == PTR_TO_BTF_ID) { reg_btf = reg->btf; - reg_ref_id = reg->btf_id; + reg_btf_id = reg->btf_id; } else { reg_btf = btf_vmlinux; - reg_ref_id = *reg2btf_ids[base_type(reg->type)]; + reg_btf_id = *reg2btf_ids[base_type(reg->type)]; } - /* Enforce strict type matching for calls to kfuncs that are acquiring - * or releasing a reference, or are no-cast aliases. We do _not_ - * enforce strict matching for kfuncs by default, + /* + * Enforce strict type matching for arguments that release a reference, + * or are no-cast aliases. We do _not_ enforce strict matching by default, * as we want to enable BPF programs to pass types that are bitwise * equivalent without forcing them to explicitly cast with something * like bpf_cast_to_kern_ctx(). @@ -12558,27 +12508,30 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ - if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || - btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) + if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || + (meta->btf && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, + arg_btf, arg_btf_id))) strict_type_match = true; - WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off)); + arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id); + arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off); + reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, ®_btf_id); + reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off); + + struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id, + reg->var_off.value, arg_btf, arg_btf_id, + strict_type_match, !type_is_alloc(reg->type)); - reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); - reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); - struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match, - !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ - taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); + taking_projection = meta->btf && btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { - verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", + verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), - btf_type_str(reg_ref_t), reg_ref_tname); + btf_type_str(arg_t), arg_tname, + reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname); return -EINVAL; } return 0; @@ -13076,7 +13029,7 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { - const char *func_name = meta->func_name, *ref_tname; + const char *func_name = meta->func_name; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); const struct btf *btf = meta->btf; @@ -13098,7 +13051,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me for (i = 0; i < nargs; i++) { struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); enum bpf_arg_type arg_type = meta->fn->arg_type[i]; - const struct btf_type *t, *ref_t; + const struct btf_type *t; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); u32 ref_id = args[i].type; @@ -13114,10 +13067,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_ptr(t)) { - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } + if (btf_type_is_ptr(t)) + btf_type_skip_modifiers(btf, t->type, &ref_id); ret = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); if (ret < 0) @@ -13433,8 +13384,8 @@ check_ok: } } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, - ref_id, meta, i, argno); + ret = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, btf, + ref_id, meta, insn_idx); if (ret < 0) return ret; break; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 14d4c1793aed..d74a9db54c9a 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -13,13 +13,13 @@ struct { const char *prog_name; const char *err_msg; } test_bpf_nf_fail_tests[] = { - { "alloc_release", "kernel function bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, - { "insert_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "lookup_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_status_after_insert", "kernel function bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, - { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, - { "change_status_after_alloc", "kernel function bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, + { "alloc_release", "bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, { "write_not_allowlisted_field", "no write support to nf_conn at off" }, { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index 0d7365b704b1..186b56357110 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -291,7 +291,7 @@ int reject_bad_type_xchg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("invalid kptr access, R2 type=trusted_ptr_prog_test_ref_kfunc") +__failure __msg("R2 must have zero offset when passed to release func") int reject_member_of_ref_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *ref_ptr; diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 8f0c45421f89..ff08aa75d6f7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -128,7 +128,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_current) } SEC("lsm.s/file_open") -__failure __msg("kernel function bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") +__failure __msg("bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") int BPF_PROG(path_d_path_kfunc_type_mismatch, struct file *file) { bpf_path_d_path((struct path *)&file->f_task_work, buf, sizeof(buf)); diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index eb6e3baef412..d730215e520b 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -152,7 +152,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "kernel function bpf_kfunc_call_memb1_release R1 expected pointer", + .errstr = "bpf_kfunc_call_memb1_release R1 expected pointer", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_memb_acquire", 1 }, { "bpf_kfunc_call_memb1_release", 5 }, diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index 1efaff296b7c..345cecc722a3 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c @@ -342,7 +342,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "invalid kptr access, R2 type=ptr_prog_test_ref_kfunc expected=ptr_prog_test_member", + .errstr = "R2 must have zero offset when passed to release func", }, { "map_kptr: ref: reference state created and released on xchg", From 2bd4a975ea5d6394dcaa6c1380653ab68252c9c7 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:10 -0700 Subject: [PATCH 115/660] bpf: Admit kfunc argument registers through check_reg_type() check_kfunc_args() open-codes exact register-type tests in most of its per-argument cases, duplicating what compatible_reg_types[] already expresses for helpers. This leaves two admission paths and prevents the helper and kfunc loops from converging. Runtime argument resolution now converts a scalar-struct BTF argument to fixed-size memory before register admission. Give the remaining kfunc-only argument kinds compatibility entries and run check_reg_type() once before the per-kind switch. Kfunc memory arguments already accept BPF-allocated objects. Normalize only the local comparison type to PTR_TO_MEM; subsequent memory checks still inspect the original register type. Keep allocated-object forms out of mem_types so helper calls continue through the existing type-mismatch path and retain its diagnostic. For ARG_PTR_TO_BTF_ID, let check_reg_type() admit BTF-backed register types and reject incompatible register classes with its standard diagnostic. Remove the now-unused lookup_reg2btf_ids(). Exact BTF identity and trust requirements remain checked later by process_arg_ptr_to_btf_id(). ARG_IGNORE and ARG_PTR_TO_PROG_AUX remain skipped because the verifier does not read those arguments from the program. Iterator arguments use the stack-pointer table. Graph nodes and ARG_PTR_TO_REFCOUNTED_KPTR share an allocated-object table. It admits owning and borrowed objects, including RCU-protected forms. Their switch cases retain API-specific ownership and BTF-record validation. ARG_PTR_TO_ALLOC_BTF_ID uses a separate table for object-drop arguments. Rename timer_types to map_value_types now that ARG_PTR_TO_WORKQUEUE and ARG_PTR_TO_TASK_WORK share it. Similarly, rename spin_lock_types to map_value_or_alloc_obj_types because graph roots and resource spin locks share its map-value-or-allocated-object admission. Moving admission checks into check_reg_type() must not discard the structured call-argument diagnostics emitted by the individual cases. Add bpf_diag_arg_type_plain() alongside bpf_diag_reg_type_plain() and use it to preserve the existing per-kind Pass suggestions where available. Other argument kinds retain the generic suggestion. The reason continues to report the actual register type and all accepted register types. Two behavior changes fall out of running admission first: - ARG_CONST_MEM_SIZE reaches process_const_arg(), and through it mark_chain_precision(), only after the register is known to be a scalar. Passing a pointer as a __szk argument used to reach backtrack_insn() with a non-scalar and trip the backtracking-misuse verifier bug. - ARG_CONST_MAP_PTR no longer needs its own type_may_be_null() test, because check_reg_type() compares whole register types. Every kfunc argument that is not explicitly ignored now passes through check_reg_type(), followed by the common register-offset check in the same order as a helper argument. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-19-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/diagnostics.c | 29 +++ kernel/bpf/diagnostics.h | 1 + kernel/bpf/verifier.c | 238 +++++------------- .../selftests/bpf/prog_tests/kfunc_call.c | 2 +- .../testing/selftests/bpf/progs/arena_kfunc.c | 2 +- .../selftests/bpf/progs/cgrp_kfunc_failure.c | 2 +- .../selftests/bpf/progs/cpumask_failure.c | 2 +- tools/testing/selftests/bpf/progs/irq.c | 4 +- tools/testing/selftests/bpf/progs/iters.c | 6 +- .../selftests/bpf/progs/iters_testmod.c | 3 +- .../bpf/progs/mem_rdonly_untrusted.c | 3 +- .../testing/selftests/bpf/progs/rbtree_fail.c | 4 +- .../bpf/progs/refcounted_kptr_fail.c | 9 +- .../selftests/bpf/progs/res_spin_lock_fail.c | 2 +- .../testing/selftests/bpf/progs/stream_fail.c | 2 +- .../selftests/bpf/progs/task_kfunc_failure.c | 2 +- .../selftests/bpf/progs/task_work_fail.c | 2 +- .../bpf/progs/test_kfunc_dynptr_param.c | 2 +- .../selftests/bpf/progs/verifier_vfs_reject.c | 2 +- .../testing/selftests/bpf/progs/wq_failures.c | 4 +- tools/testing/selftests/bpf/verifier/calls.c | 8 +- 21 files changed, 120 insertions(+), 209 deletions(-) diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 0abbbe177e31..a2cac59c6639 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -960,6 +960,35 @@ const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_t } } +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_MEM_SIZE: + case ARG_CONST_MEM_SIZE: + return "an integer scalar length for this memory argument"; + case ARG_PTR_TO_CTX: + return "the original program context pointer or preserve it before modifying registers"; + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return "an integer scalar value for this argument, not a pointer or resource object"; + case ARG_PTR_TO_CONST_STR: + return "a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value"; + case ARG_PTR_TO_DYNPTR: + return "the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path"; + case ARG_PTR_TO_ALLOC_BTF_ID: + return "a pointer returned by the matching BPF object allocation path"; + case ARG_PTR_TO_REFCOUNTED_KPTR: + return "an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field"; + case ARG_PTR_TO_ITER: + return "the address of a stack iterator object for iterator new, next, and destroy calls"; + case ARG_PTR_TO_IRQ_FLAG: + return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()"; + default: + return "a value with one of the accepted pointer or scalar types for this call"; + } +} + static const char *diag_arg_ordinal(int argno) { switch (argno) { diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d1b79945008a..a4102fb049ec 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -51,6 +51,7 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 11d28c8d9c79..254458cc1737 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5166,18 +5166,6 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) -{ - enum bpf_reg_type type; - - for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { - if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) - return type; - } - - return NOT_INIT; -} - static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -8246,7 +8234,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, static int resolve_func_arg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 arg, - struct bpf_call_arg_meta *meta, int insn_idx, + struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size); static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type, @@ -8296,7 +8284,7 @@ static const struct bpf_reg_types mem_types = { }, }; -static const struct bpf_reg_types spin_lock_types = { +static const struct bpf_reg_types map_value_or_alloc_obj_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, @@ -8325,7 +8313,30 @@ static const struct bpf_reg_types percpu_btf_ptr_types = { static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; -static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types arena_types = { + .types = { + PTR_TO_ARENA, + SCALAR_VALUE, + } +}; + +static const struct bpf_reg_types alloc_obj_drop_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU, + } +}; + +static const struct bpf_reg_types alloc_obj_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, + } +}; + static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, @@ -8356,16 +8367,30 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { #endif [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, - [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, + [ARG_PTR_TO_SPIN_LOCK] = &map_value_or_alloc_obj_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, - [ARG_PTR_TO_TIMER] = &timer_types, + [ARG_PTR_TO_TIMER] = &map_value_types, [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, [ARG_PTR_TO_DYNPTR] = &dynptr_types, + [ARG_CONST_SCALAR] = &scalar_types, + [ARG_CONST_MEM_SIZE] = &scalar_types, + [ARG_PTR_TO_ALLOC_BTF_ID] = &alloc_obj_drop_types, + [ARG_PTR_TO_REFCOUNTED_KPTR] = &alloc_obj_types, + [ARG_PTR_TO_ITER] = &stack_ptr_types, + [ARG_PTR_TO_LIST_HEAD] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_LIST_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RB_ROOT] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_RB_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_WORKQUEUE] = &map_value_types, + [ARG_PTR_TO_TASK_WORK] = &map_value_types, + [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, + [ARG_PTR_TO_ARENA] = &arena_types, }; static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, @@ -8466,6 +8491,9 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; } + if (meta->btf && base_type(arg_type) == ARG_PTR_TO_BTF_ID && + (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) + goto found; /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY @@ -8485,6 +8513,10 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; + /* Allow allocated memory for kfunc ARG_PTR_TO_MEM but not helper. */ + if (meta->btf && base_type(arg_type) == ARG_PTR_TO_MEM && + type_is_ptr_alloc_obj(type)) + type = PTR_TO_MEM; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && @@ -8509,7 +8541,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); accepted = bpf_diag_expected_reg_types(env, compatible->types, i); bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, - "Pass a value with one of the accepted pointer or scalar types for this call.", + bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)), "it has type %s, but this argument accepts %s", actual, accepted); return -EACCES; @@ -8824,7 +8856,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EACCES; } - err = resolve_func_arg_type(env, reg, arg, meta, insn_idx, &arg_type, &arg_size); + err = resolve_func_arg_type(env, reg, arg, meta, &arg_type, &arg_size); if (err) return err; @@ -11916,7 +11948,7 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, static int resolve_func_arg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 arg, - struct bpf_call_arg_meta *meta, int insn_idx, + struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size) { argno_t argno = argno_from_arg(arg + 1); @@ -11936,30 +11968,15 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)]) return 0; + args = btf_params(meta->func_proto); ref_id = *meta->fn->arg_btf_id[arg]; btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - if (!btf_type_is_scalar_struct(env, btf, ref_t)) { - enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); - const char *expected_type; - - verbose(env, "%s is %s expected %s %s", - reg_arg_name(env, argno), reg_type_str(env, reg->type), - btf_type_str(ref_t), ref_tname); - if (reg2btf_type != NOT_INIT) - verbose(env, " or %s", reg_type_str(env, reg2btf_type)); - verbose(env, "\n"); - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, - "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", - "the function expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", - expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } + if (!btf_type_is_scalar_struct(env, btf, ref_t)) + return 0; resolve_ret = btf_resolve_size(btf, ref_t, arg_size); if (IS_ERR(resolve_ret)) { @@ -13101,25 +13118,20 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) ref_id = *meta->fn->arg_btf_id[i]; - ret = resolve_func_arg_type(env, reg, i, meta, insn_idx, &arg_type, &arg_size); + ret = resolve_func_arg_type(env, reg, i, meta, &arg_type, &arg_size); if (ret < 0) return ret; + + ret = check_reg_type(env, reg, argno, arg_type, meta); + if (ret < 0) + return ret; + ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) return ret; switch (base_type(arg_type)) { case ARG_CONST_SCALAR: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = process_const_arg(env, reg, argno, meta); if (ret < 0) { if (ret == -EINVAL) @@ -13131,27 +13143,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } break; case ARG_SCALAR: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) meta->r0_rdonly = true; ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); @@ -13165,17 +13158,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } break; case ARG_PTR_TO_CTX: - if (reg->type != PTR_TO_CTX) { - verbose(env, "%s expected pointer to ctx, but got %s\n", - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the original program context pointer or preserve it before modifying registers.", - "the kfunc expects a context pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); if (ret < 0) @@ -13184,11 +13166,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } break; case ARG_PTR_TO_ARENA: - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a pointer to arena or scalar\n", - reg_arg_name(env, argno)); - return -EINVAL; - } break; case ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { @@ -13203,15 +13180,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me reg_arg_name(env, argno)); return -EINVAL; } - } else { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer returned by the matching BPF object allocation path.", - "the kfunc expects an allocated object pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); @@ -13260,12 +13228,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; break; case ARG_PTR_TO_LIST_HEAD: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); @@ -13276,12 +13238,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; break; case ARG_PTR_TO_RB_ROOT: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); @@ -13341,12 +13297,6 @@ check_ok: return ret; break; case ARG_CONST_MAP_PTR: - if (base_type(reg->type) != CONST_PTR_TO_MAP || - type_may_be_null(reg->type)) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } ret = process_map_ptr_arg(env, reg, argno, meta); if (ret < 0) return ret; @@ -13435,16 +13385,6 @@ check_ok: argno_t buff_argno = argno_from_arg(i); enum bpf_mem_size_failure failure; - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar length for this memory argument.", - "the kfunc expects a scalar memory size, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (bpf_register_is_null(buff_reg)) break; @@ -13482,23 +13422,9 @@ check_ok: break; } case ARG_PTR_TO_FUNC: - if (reg->type != PTR_TO_FUNC) { - verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); - return -EINVAL; - } meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_REFCOUNTED_KPTR: - if (!type_is_ptr_alloc_obj(reg->type)) { - verbose(env, "%s is neither owning or non-owning ref\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", - "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) meta->arg_owning_ref = true; @@ -13518,61 +13444,26 @@ check_ok: meta->arg_btf_id = reg->btf_id; break; case ARG_PTR_TO_CONST_STR: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a const string\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", - "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } ret = check_arg_const_str(env, reg, argno); if (ret) return ret; break; case ARG_PTR_TO_WORKQUEUE: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); if (ret < 0) return ret; break; case ARG_PTR_TO_TIMER: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; case ARG_PTR_TO_TASK_WORK: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); if (ret < 0) return ret; break; case ARG_PTR_TO_IRQ_FLAG: - if (reg->type != PTR_TO_STACK) { - verbose(env, "%s doesn't point to an irq flag on stack\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", - "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } ret = process_irq_flag(env, reg, argno, meta); if (ret < 0) return ret; @@ -13585,13 +13476,6 @@ check_ok: verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } - - if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s doesn't point to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) return -EFAULT; if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 2b39cc1b09f9..0063e60d6f2f 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -70,7 +70,7 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), - TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 type=scalar expected=ctx"), TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 50609f3b0564..6578cf12fa27 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -205,7 +205,7 @@ int arena_arg_no_arena(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("is not a pointer to arena or scalar") +__failure __msg("R1 type=fp expected=arena, scalar") int arena_arg_bad_reg(void *ctx) { u64 buf = 0; diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index efe7bcae70f8..a7c8c765a98d 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 is fp expected STRUCT cgroup") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 6d730535eb95..76a3cba6f23c 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -243,7 +243,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl } SEC("tp_btf/task_newtask") -__failure __msg("leads to invalid memory access") +__failure __msg("R2 type=scalar expected=fp") int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags) { void *garbage = (void *)0x123456; diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c index a4a007866a33..53df6d248e26 100644 --- a/tools/testing/selftests/bpf/progs/irq.c +++ b/tools/testing/selftests/bpf/progs/irq.c @@ -15,7 +15,7 @@ struct bpf_res_spin_lock lockA __hidden SEC(".data.A"); struct bpf_res_spin_lock lockB __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("R1 type=map_value expected=fp") int irq_save_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_save(&global_flags); @@ -23,7 +23,7 @@ int irq_save_bad_arg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("R1 type=map_value expected=fp") int irq_restore_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_restore(&global_flags); diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index c6699159dacd..65d4c6e01f93 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -1688,7 +1688,7 @@ int iter_subprog_check_stacksafe(const void *ctx) struct bpf_iter_num global_it; SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_new_bad_arg(const void *ctx) { bpf_iter_num_new(&global_it, 0, 1); @@ -1696,7 +1696,7 @@ int iter_new_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_next_bad_arg(const void *ctx) { bpf_iter_num_next(&global_it); @@ -1704,7 +1704,7 @@ int iter_next_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_destroy_bad_arg(const void *ctx) { bpf_iter_num_destroy(&global_it); diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 5a3ff65e8234..f65cc9766633 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -105,8 +105,7 @@ out: } SEC("raw_tp/sys_enter") -__failure __msg("R1 cannot write into rdonly_mem") -/* Message should not be 'R1 cannot write into rdonly_trusted_mem' */ +__failure __msg("R1 type=rdonly_mem expected=fp") int iter_next_ptr_mem_not_trusted(const void *ctx) { struct bpf_iter_num num_it; diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 3e0d4f687aaa..23019023511a 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -118,8 +118,7 @@ int atomic_rmw_not_ok(void *ctx) SEC("socket") __failure -__msg("invalid access to memory, mem_size=0 off=0 size=4") -__msg("R1 min value is outside of the allowed memory range") +__msg("R1 type=rdonly_untrusted_mem expected=fp") int kfunc_param_not_ok(void *ctx) { int *p; diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c index 4504608196ab..08709f23ec0f 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_fail.c +++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c @@ -180,7 +180,7 @@ err_out: } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_add_release_unlock_escape(void *ctx) { struct node_data *n; @@ -204,7 +204,7 @@ long rbtree_api_add_release_unlock_escape(void *ctx) } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_first_release_unlock_escape(void *ctx) { struct bpf_rb_node *res; diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 338e43822ffe..e80f78fae227 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -118,8 +118,8 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") -__failure __msg("R1 is neither owning or non-owning ref") -__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +__failure __msg("R1 type=ctx expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") +__msg("type ctx, but this argument accepts ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_non_object(void *ctx) { return bpf_refcount_acquire(ctx) != NULL; @@ -159,8 +159,7 @@ long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx) SEC("?syscall") __failure -__msg("bpf_rbtree_remove can only take non-owning or refcounted " - "bpf_rb_node pointer") +__msg("R2 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_remove_after_rcu_unlock(void *ctx) { struct map_value_rcu_graph *mapval; @@ -190,7 +189,7 @@ long rbtree_remove_after_rcu_unlock(void *ctx) } SEC("?syscall") -__failure __msg("R1 is neither owning or non-owning ref") +__failure __msg("R1 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_after_rcu_unlock(void *ctx) { struct map_value_refcount_only *mapval; diff --git a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c index 330682a88c16..8fd591bd1f6c 100644 --- a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c +++ b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c @@ -24,7 +24,7 @@ struct bpf_spin_lock lock __hidden SEC(".data.A"); struct bpf_res_spin_lock res_lock __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("point to map value or allocated object") +__failure __msg("R1 type=untrusted_ptr_ expected=map_value, ptr_") int res_spin_lock_arg(struct __sk_buff *ctx) { struct arr_elem *elem; diff --git a/tools/testing/selftests/bpf/progs/stream_fail.c b/tools/testing/selftests/bpf/progs/stream_fail.c index 21428bb1ee59..10ebb4a7f105 100644 --- a/tools/testing/selftests/bpf/progs/stream_fail.c +++ b/tools/testing/selftests/bpf/progs/stream_fail.c @@ -23,7 +23,7 @@ int stream_vprintk_scalar_arg(void *ctx) } SEC("syscall") -__failure __msg("R2 doesn't point to a const string") +__failure __msg("R2 type=ctx expected=map_value") int stream_vprintk_string_arg(void *ctx) { bpf_stream_vprintk(BPF_STDOUT, ctx, NULL, 0); diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 9979766d4d74..404f7f9d7150 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; diff --git a/tools/testing/selftests/bpf/progs/task_work_fail.c b/tools/testing/selftests/bpf/progs/task_work_fail.c index 3186e7b4b24e..bc56bdaca780 100644 --- a/tools/testing/selftests/bpf/progs/task_work_fail.c +++ b/tools/testing/selftests/bpf/progs/task_work_fail.c @@ -58,7 +58,7 @@ int mismatch_map(struct pt_regs *args) } SEC("perf_event") -__failure __msg("R2 doesn't point to a map value") +__failure __msg("R2 type=fp expected=map_value") int no_map_task_work(struct pt_regs *args) { struct task_struct *task; diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c index bf48fc43c7ab..f7a83e502454 100644 --- a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c @@ -40,7 +40,7 @@ int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, } SEC("?lsm.s/bpf") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=map_value expected=fp, dynptr_ptr") int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel) { static struct bpf_dynptr val; diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index ff08aa75d6f7..2cea3d9c3647 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; diff --git a/tools/testing/selftests/bpf/progs/wq_failures.c b/tools/testing/selftests/bpf/progs/wq_failures.c index 32dc8827e128..bd30217579d4 100644 --- a/tools/testing/selftests/bpf/progs/wq_failures.c +++ b/tools/testing/selftests/bpf/progs/wq_failures.c @@ -48,7 +48,7 @@ __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __failure __msg(": (85) call bpf_wq_init#") /* anchor message */ -__msg("pointer in R2 isn't map pointer") +__msg("R2 type=fp expected=map_ptr") long test_wq_init_nomap(void *ctx) { struct bpf_wq *wq; @@ -98,7 +98,7 @@ __failure * is a correct bpf_wq pointer. */ __msg(": (85) call bpf_wq_set_callback#") /* anchor message */ -__msg("R1 doesn't point to a map value") +__msg("R1 type=fp expected=map_value") long test_wrong_wq_pointer(void *ctx) { int key = 0; diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index d730215e520b..8b94b87135bc 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail1", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail3", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -76,7 +76,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 expected pointer to ctx, but got fp", + .errstr = "R1 type=fp expected=ctx", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_pass_ctx", 2 }, }, From 8fe994c80af271979cf2b3e830d4d3f7903edd50 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:11 -0700 Subject: [PATCH 116/660] bpf: Consolidate function call pkt_access validation check_func_arg() checks whether a helper permits packet pointers before dispatching argument-specific memory validation. check_kfunc_args() has no equivalent check, even though kfunc memory arguments may be backed by packet data. Move packet-access validation to check_helper_mem_access(), where the access direction is known and helper, kfunc, and global-subprogram memory arguments converge. Pass call metadata there so helpers continue to require bpf_func_proto::pkt_access, while writes through kfunc and global subprogram arguments use the program-type policy and set env->seen_direct_write. Keep call metadata when variable-size memory disables raw mode by clearing arg_raw_mem.regno instead, and pass it through the map-key path as well. This also makes kfunc and global-subprogram packet writes request the required writable-packet prologue and rejects them for program types that only support direct packet reads. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-20-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 24 +++++++++---------- .../bpf/progs/verifier_helper_packet_access.c | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 254458cc1737..bb33d0755a61 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4881,7 +4881,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_func_proto *fn, + const struct bpf_call_arg_meta *meta, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4905,10 +4905,11 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (fn) - return fn->pkt_access; + if (meta && !meta->btf && meta->func_id) + return meta->fn->pkt_access; - env->seen_direct_write = true; + if (t == BPF_WRITE) + env->seen_direct_write = true; return true; case BPF_PROG_TYPE_CGROUP_SOCKOPT: @@ -7095,6 +7096,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: + if (!may_access_direct_pkt_data(env, meta, access_type)) { + verbose(env, "function access to the packet is not allowed\n"); + return -EACCES; + } return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: @@ -7211,7 +7216,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, * the memory that the helper could just partially fill up. */ if (!tnum_is_const(size_reg->var_off)) - meta = NULL; + meta->arg_raw_mem.regno = 0; if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -8822,7 +8827,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, arg); enum bpf_arg_type arg_type = fn->arg_type[arg]; int regno = reg_from_argno(argno); - enum bpf_reg_type type = reg->type; u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; @@ -8850,12 +8854,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return 0; } - if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, fn, BPF_READ)) { - verbose(env, "helper access to the packet is not allowed\n"); - return -EACCES; - } - err = resolve_func_arg_type(env, reg, arg, meta, &arg_type, &arg_size); if (err) return err; @@ -8932,7 +8930,7 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta, NULL); if (err) return err; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c index 71cee3f58324..12786b72c694 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c @@ -258,7 +258,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test11, cls unsuitable helper 1") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test11_cls_unsuitable_helper_1(void) { asm volatile (" \ @@ -283,7 +283,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test12, cls unsuitable helper 2") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test12_cls_unsuitable_helper_2(void) { asm volatile (" \ From 5647b77587ce164008c54cb178367e587d6f6bcb Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:12 -0700 Subject: [PATCH 117/660] selftests/bpf: Test kfunc packet memory direct writes A kfunc memory argument may be backed by packet data. Exercise this with bpf_skb_ct_lookup(), which writes errors through its opts memory argument. Verify that the verifier marks the program as directly writing packet data, causing TC to emit its writable-skb prologue. Match the semantic prologue sequence while allowing architecture-specific zero extensions and the endian-specific cloned-bit mask. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-21-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_kfunc_packet_access.c | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index f7f94ccebce2..b97381448248 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -54,6 +54,7 @@ #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_packet_access.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" @@ -218,6 +219,7 @@ void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_packet_access(void) { RUN_TESTS(verifier_kfunc_packet_access); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c new file mode 100644 index 000000000000..88009566d92f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" + +struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, + struct bpf_sock_tuple *bpf_tuple, + u32 tuple__sz, struct bpf_ct_opts *opts, + u32 opts__sz) __ksym; +void bpf_ct_release(struct nf_conn *nfct) __ksym; + +char _license[] SEC("license") = "GPL"; + +SEC("tc") +__description("kfunc packet write requests writable skb") +__success +/* bpf_unclone_prologue() */ +__xlated("r6 = *(u8 *)(r1 +{{[0-9]+}})") +__xlated("...") +__xlated("w6 &= {{(1|128)}}") +__xlated("...") +__xlated("if r6 == 0x0 goto") +__xlated("r6 = r1") +__xlated("r2 ^= r2") +__xlated("call") +__xlated("if r0 == 0x0 goto") +__xlated("w0 = 2") +__xlated("...") +__xlated("exit") +__xlated("r1 = r6") +int kfunc_packet_write(struct __sk_buff *skb) +{ + void *data_end = (void *)(long)skb->data_end; + void *data = (void *)(long)skb->data; + struct bpf_sock_tuple tuple = {}; + struct nf_conn *nfct; + + if (data + sizeof(struct bpf_ct_opts) > data_end) + return 0; + + /* An invalid tuple size makes bpf_skb_ct_lookup() write opts->error. */ + nfct = bpf_skb_ct_lookup(skb, &tuple, 1, data, sizeof(struct bpf_ct_opts)); + if (nfct) + bpf_ct_release(nfct); + return 0; +} From 5dc1549afac9c5d82bdde9ac4e000c859bbbf991 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:13 -0700 Subject: [PATCH 118/660] bpf: Consolidate release argument validation Helper and kfunc argument verification both require a live owning reference for non-dynptr release arguments. Whether that reference may be NULL is expressed by PTR_MAYBE_NULL in helper prototypes and the __nullable BTF suffix for kfuncs. Factor the shared ownership policy into check_func_arg_release(). Drop the helper-only rejection of maybe-null owning references. This allows bpf_kptr_xchg() to accept its explicitly nullable source directly, matching nullable kfunc release arguments. The verifier consumes the reference regardless of its runtime nullness. Update the tests so an owned nullable source succeeds while a nullable non-owning source remains rejected for lacking ownership. Preserve the identity of a nullable per-CPU allocation when bpf_kptr_xchg() transfers its ownership inside an RCU critical section. The converted MEM_RCU aliases no longer own a reference, but still need a non-zero ID so a NULL check on one allocation cannot refine another unrelated allocation. Use common legacy and structured diagnostic text for both call kinds. Tailor the structured suggestion to describe a pointer for helpers and include the expected BTF type for kfuncs. Resolve that type lazily from the kfunc BTF parameter only when reporting an ownership failure. The kfunc path now keys the check directly on the cached OBJ_RELEASE argument flag instead of comparing the argument register against meta->release_regno, and uses the cached argument kind for the dynptr exemption. This is another prerequisite for routing both call types through check_func_arg(). Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-22-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 82 ++++++++++--------- .../selftests/bpf/prog_tests/cb_refs.c | 4 +- .../selftests/bpf/progs/cgrp_kfunc_failure.c | 6 +- .../selftests/bpf/progs/cpumask_failure.c | 2 +- .../selftests/bpf/progs/map_kptr_fail.c | 4 +- .../selftests/bpf/progs/task_kfunc_failure.c | 6 +- .../bpf/progs/verifier_global_ptr_args.c | 2 +- .../bpf/progs/verifier_ref_tracking.c | 2 +- .../selftests/bpf/progs/verifier_sock.c | 6 +- .../selftests/bpf/progs/verifier_vfs_reject.c | 2 +- .../selftests/bpf/progs/wakeup_source_fail.c | 2 +- 11 files changed, 63 insertions(+), 55 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bb33d0755a61..fe8a45303602 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8463,6 +8463,43 @@ static int check_func_arg_nullability(struct bpf_verifier_env *env, return -EACCES; } +static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (!arg_type_is_release(arg_type)) + return 0; + + if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) || + bpf_register_is_null(reg)) + return 0; + + verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", + meta->func_name, reg_arg_name(env, argno)); + + if (meta->btf) { + const struct btf_param *btf_arg; + const struct btf_type *t; + u32 ref_id; + + btf_arg = &btf_params(meta->func_proto)[arg_idx_from_argno(argno)]; + ref_id = btf_arg->type; + t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); + if (btf_type_is_ptr(t)) + btf_type_skip_modifiers(meta->btf, t->type, &ref_id); + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, ref_id)); + } + + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.", + expected_type), + "release functions require a value that owns a live resource returned by a matching acquire function"); + return -EINVAL; +} + static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, const enum bpf_reg_type *types, int count) { @@ -8881,29 +8918,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; skip_type_check: - if (arg_type_is_release(arg_type)) { - if (type_may_be_null(reg->type)) { - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, meta->func_name, - "the pointer may be NULL, but this call requires a non-NULL pointer", - "Add a NULL check and make the call only on the non-NULL path."); - return -EACCES; - } - - if (!arg_type_is_dynptr(arg_type) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, - "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - meta->func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, meta->func_name, - "release helpers require a value that owns a live resource returned by a matching acquire helper", - "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); - return -EINVAL; - } - } + err = check_func_arg_release(env, reg, argno, arg_type, meta, insn_idx); + if (err) + return err; if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); @@ -9748,7 +9765,8 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->id = 0; + if (!type_may_be_null(reg->type)) + reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; bpf_diag_mod_end(env); @@ -13089,19 +13107,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; - if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", - "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", - expected_type); - return -EINVAL; - } + ret = check_func_arg_release(env, reg, argno, arg_type, meta, insn_idx); + if (ret < 0) + return ret; if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index 50ea3d72d446..e415acc08b16 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -11,8 +11,8 @@ struct { const char *prog_name; const char *err_msg; } cb_refs_tests[] = { - { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, - { "leak_prog", "Possibly NULL pointer passed to trusted R2" }, + { "underflow_prog", "release function bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, + { "leak_prog", "Unreleased reference id=4 alloc_insn=3" }, /* alloc_insn=3{2,3} */ { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ }; diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index a7c8c765a98d..731eaa09db96 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -154,7 +154,7 @@ int BPF_PROG(cgrp_kfunc_xchg_unreleased, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_rcu_get_release, struct cgroup *cgrp, const char *path) { struct cgroup *kptr; @@ -191,7 +191,7 @@ int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired = (struct cgroup *)&path; @@ -237,7 +237,7 @@ int BPF_PROG(cgrp_kfunc_release_null, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_unacquired, struct cgroup *cgrp, const char *path) { /* Cannot release trusted cgroup pointer which was not acquired. */ diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 76a3cba6f23c..c89c88db39d1 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -183,7 +183,7 @@ int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int BPF_PROG(test_global_mask_rcu_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *prev, *curr; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index 186b56357110..ac4003bfb8b0 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -252,7 +252,7 @@ int reject_untrusted_store_to_ref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("release helper bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int reject_untrusted_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; @@ -364,7 +364,7 @@ int kptr_xchg_ref_state(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to trusted R2") +__success int kptr_xchg_possibly_null(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 404f7f9d7150..0ca2e1a38a94 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -179,7 +179,7 @@ int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired = (struct task_struct *)&clone_flags; @@ -225,7 +225,7 @@ int BPF_PROG(task_kfunc_release_null, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_unacquired, struct task_struct *task, u64 clone_flags) { /* Cannot release trusted task pointer which was not acquired. */ @@ -353,7 +353,7 @@ int BPF_PROG(task_access_comm4, struct task_struct *task, const char *buf, bool } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flags) { struct task_struct *local; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index a3d2af8dc839..b277b1efb8c7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -177,7 +177,7 @@ __weak int subprog_trusted_destroy(struct task_struct *task __arg_trusted) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(trusted_destroy_fail, struct task_struct *task, u64 clone_flags) { return subprog_trusted_destroy(task); diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c index f3fad911b5bc..799db6f5713b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c @@ -1288,7 +1288,7 @@ l1_%=: r1 = r6; \ SEC("tc") __description("reference tracking: bpf_sk_release(listen_sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_listen_sk(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index d59b2f905671..2a136c917680 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -603,7 +603,7 @@ l2_%=: r0 = *(u32*)(r0 + %[bpf_tcp_sock_snd_cwnd]); \ SEC("tc") __description("bpf_sk_release(skb->sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_skb_sk(void) { asm volatile (" \ @@ -620,7 +620,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("bpf_sk_release(bpf_sk_fullsock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_fullsock_skb_sk(void) { asm volatile (" \ @@ -644,7 +644,7 @@ l1_%=: r1 = r0; \ SEC("tc") __description("bpf_sk_release(bpf_tcp_sock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_tcp_sock_skb_sk(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 2cea3d9c3647..b5f456d57669 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -80,7 +80,7 @@ int BPF_PROG(get_task_exe_file_kfunc_unreleased) } SEC("lsm.s/file_open") -__failure __msg("release kfunc bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(put_file_kfunc_unacquired, struct file *file) { /* Can't release an unacquired pointer. */ diff --git a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c index d4d0f1610853..ff9ea5aa80ad 100644 --- a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c +++ b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c @@ -42,7 +42,7 @@ int wakeup_source_access_lock_fields(void *ctx) } SEC("syscall") -__failure __msg("release kfunc bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") int wakeup_source_unlock_no_lock(void *ctx) { struct bpf_ws_lock *lock = (void *)0x1; From 4f18de58a44cc15e2ec18c07bb1897d97f9dc1a8 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:14 -0700 Subject: [PATCH 119/660] selftests/bpf: Test nullable per-CPU kptr identity after exchange Pass two unchecked bpf_percpu_obj_new() results to bpf_kptr_xchg() inside an RCU critical section. The exchanges consume ownership and leave nullable RCU-protected aliases. Checking the first alias must not make the second non-NULL. Verify that bpf_this_cpu_ptr() still rejects the unchecked second alias. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-23-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/percpu_alloc_fail.c | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c index 3701f4ea58c7..57615b0f0c25 100644 --- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c +++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c @@ -24,6 +24,7 @@ struct val_600b_t { struct elem { long sum; struct val_t __percpu_kptr *pc; + struct val_t __percpu_kptr *pc2; }; struct { @@ -46,6 +47,8 @@ struct { struct task_struct *bpf_task_from_pid(s32 pid) __ksym; void bpf_task_release(struct task_struct *p) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; long ret; @@ -123,6 +126,38 @@ int BPF_PROG(test_array_map_3) return 0; } +SEC("?fentry.s/bpf_fentry_test1") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(reject_nullable_percpu_xchg_alias) +{ + struct val_t __percpu_kptr *p1, *p2, *old; + struct val_t *v; + struct elem *e; + int index = 0; + + e = bpf_map_lookup_elem(&array, &index); + if (!e) + return 0; + + p1 = bpf_percpu_obj_new(struct val_t); + p2 = bpf_percpu_obj_new(struct val_t); + + bpf_rcu_read_lock(); + old = bpf_kptr_xchg(&e->pc, p1); + if (old) + bpf_percpu_obj_drop(old); + old = bpf_kptr_xchg(&e->pc2, p2); + if (old) + bpf_percpu_obj_drop(old); + + if (p1) { + v = bpf_this_cpu_ptr(p2); + v->b = 1; + } + bpf_rcu_read_unlock(); + return 0; +} + SEC("?fentry.s/bpf_fentry_test1") __failure __msg("R1 expected for bpf_percpu_obj_drop()") int BPF_PROG(test_array_map_4) From ab7fff217fd14ec1b3e1a25665ba282f8c91ed6a Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 11 Sep 2026 15:04:15 -0700 Subject: [PATCH 120/660] bpf: Check helper and kfunc arguments in one path Generated kfunc prototypes now carry the argument kinds and metadata needed by the helper argument checker, but kfunc calls still duplicate the per-argument loop and switch. Add check_func_args() to determine the argument count, validate outgoing kfunc stack arguments, and drive check_func_arg() for both call kinds. Move the remaining kfunc-only argument handlers, including the dynptr clone runtime refinement, into check_func_arg(), derive each kfunc BTF referent there once, and skip check_reg_arg() for stack-passed arguments that check_outgoing_stack_args() already validated. Keep special-kfunc ID checks behind small helpers so their table can remain in place. Keep the intentional differences selected by call metadata: helpers retain their privileged ARG_ANYTHING pointer-leak behavior, while kfunc scalars use strict ARG_SCALAR register admission. The shared ARG_PTR_TO_BTF_ID case retains helper-only compatible-ID and bpf_kptr_xchg() handling, performs kfunc trusted/RCU provenance checks, and selects the expected BTF based on call kind. Helper and kfunc memory arguments likewise retain their access modes and share fixed-memory and memory/size diagnostics. Treat an accepted nullable NULL as a completed argument check, matching the former kfunc continue path. The skipped helper-specific processing is either a no-op for NULL or deferred to the paired size argument. Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260911220415.1396439-24-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 932 +++++++++--------- .../selftests/bpf/prog_tests/cb_refs.c | 2 +- .../selftests/bpf/progs/cgrp_kfunc_failure.c | 2 +- .../selftests/bpf/progs/task_kfunc_failure.c | 2 +- .../selftests/bpf/progs/wakeup_source_fail.c | 2 +- 5 files changed, 445 insertions(+), 495 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fe8a45303602..617a277c3558 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7719,7 +7719,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Dynptr has to be an uninitialized dynptr\n"); bpf_diag_res( env, insn_idx, "dynptr is already initialized", - "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7848,8 +7848,8 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * reg_arg_name(env, argno)); bpf_diag_call_arg( env, insn_idx, argno, meta->func_name, - "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", - "Pass the exact iterator state type expected by this kfunc."); + "the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this function."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -8245,6 +8245,39 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re argno_t argno, enum bpf_arg_type arg_type, const struct btf *arg_btf, u32 arg_btf_id, struct bpf_call_arg_meta *meta, int insn_idx); +static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, + const struct btf_param *arg); +static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, + const struct btf_param *arg, + const char *name); +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_obj_drop_kfunc(u32 func_id); +static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); +static bool is_bpf_rbtree_add_kfunc(u32 func_id); +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta); +static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); +static int process_irq_flag(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env); struct bpf_reg_types { const enum bpf_reg_type types[10]; @@ -8813,7 +8846,6 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } static bool can_elide_value_nullness(const struct bpf_map *map); -static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_call_arg_meta *meta) @@ -8857,6 +8889,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, int insn_idx) { + const struct btf_param *btf_arg = meta->btf ? &btf_params(meta->func_proto)[arg] : NULL; const struct bpf_func_proto *fn = meta->fn; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); @@ -8882,6 +8915,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } + /* Preserve the legacy helper behavior for privileged pointer leaks. */ if (arg_type == ARG_ANYTHING) { if (__is_pointer_value(env->allow_ptr_leaks, reg)) { verbose(env, "%s leaks addr into helper function\n", @@ -8896,13 +8930,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - /* A NULL register has a SCALAR_VALUE type, so skip - * type checking. - */ - err = mark_chain_precision(env, regno); + err = mark_arg_precision(env, argno); if (err) return err; - goto skip_type_check; + return 0; } err = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); @@ -8917,7 +8948,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; -skip_type_check: err = check_func_arg_release(env, reg, argno, arg_type, meta, insn_idx); if (err) return err; @@ -8926,6 +8956,19 @@ skip_type_check: update_ref_obj(&meta->ref_obj, reg); switch (base_type(arg_type)) { + case ARG_CONST_SCALAR: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + break; + case ARG_SCALAR: + break; case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ err = process_map_ptr_arg(env, reg, argno, meta); @@ -8991,6 +9034,7 @@ skip_type_check: case ARG_PTR_TO_BTF_ID_SOCK_COMMON: { const u32 *arg_btf_id = fn->arg_btf_id[arg]; + const struct btf *arg_btf = meta->btf ?: btf_vmlinux; if (!meta->btf) { const struct bpf_reg_types *compatible; @@ -9018,7 +9062,39 @@ skip_type_check: } } - err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, btf_vmlinux, + if (meta->btf && (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type))) { + if (!(arg_type & MEM_RCU)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.", + "the function requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Use this function with a pointer that is valid in an RCU read lock region.", + "the function requires an RCU-protected pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + } + + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf, *arg_btf_id, meta, insn_idx); if (err < 0) return err; @@ -9055,44 +9131,274 @@ skip_type_check: if (err) return err; break; + case ARG_PTR_TO_CTX: + if (is_bpf_cast_to_kern_ctx_kfunc(meta)) { + err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); + if (err < 0) + return -EINVAL; + meta->ret_btf_id = err; + } + break; + case ARG_PTR_TO_ARENA: + break; + case ARG_PTR_TO_ALLOC_BTF_ID: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { + if (!is_bpf_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { + if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_percpu_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (meta->btf == btf_vmlinux) { + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + } + break; case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_MEM: + { + enum bpf_access_type access_type; + bool known_memory; + /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno, arg_size, - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); - if (err) - return err; - if (arg_type & MEM_ALIGNED) - err = check_ptr_alignment(env, reg, 0, arg_size, true); + if (!(arg_type & MEM_FIXED_SIZE)) + break; + + access_type = arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ; + if (meta->btf) + access_type = BPF_READ | BPF_WRITE; + + err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); + if (err < 0) { + if (known_memory) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass memory with at least the required number of accessible bytes and suitable read or write access.", + "the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the function expects %u bytes of memory, but it is %s and not verifier-known memory", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + return err; + } + if (arg_type & MEM_ALIGNED) + err = check_ptr_alignment(env, reg, 0, arg_size, true); + break; + } + case ARG_CONST_MEM_SIZE: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + fallthrough; + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, arg - 1); + argno_t buff_argno = argno_from_arg(arg); + enum bpf_mem_size_failure failure; + const char *buff_arg, *size_arg; + bool zero_size_allowed; + u32 access_type; + + if (meta->btf && bpf_register_is_null(buff_reg)) + break; + + access_type = fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ; + if (meta->btf) + access_type = BPF_READ | BPF_WRITE; + + zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; + + err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno, + access_type, zero_size_allowed, meta, &failure); + if (!err) + break; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(reg) < 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(reg)); + else if (!zero_size_allowed && reg_umin(reg) == 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Ensure the memory size is non-zero before this call.", + "the memory size in %s may be zero, but the function requires a non-zero size", + size_arg); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ); } break; - case ARG_MEM_SIZE: - err = check_mem_size_reg(env, get_func_arg_reg(caller, regs, arg - 1), reg, - argno_from_arg(arg), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta, NULL); - break; - case ARG_MEM_SIZE_OR_ZERO: - err = check_mem_size_reg(env, get_func_arg_reg(caller, regs, arg - 1), reg, - argno_from_arg(arg), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta, NULL); - break; - case ARG_PTR_TO_DYNPTR: + } + case ARG_PTR_TO_DYNPTR: { + if (is_bpf_dynptr_clone_kfunc(meta) && + (arg_type & MEM_UNINIT)) { + enum bpf_dynptr_type parent_type = meta->dynptr.type; + + if (parent_type == BPF_DYNPTR_TYPE_INVALID) { + verifier_bug(env, "no dynptr type for parent of clone"); + return -EFAULT; + } + + arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); + } + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; break; - case ARG_CONST_ALLOC_SIZE_OR_ZERO: - err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (err) + } + case ARG_PTR_TO_ITER: + if (is_bpf_iter_css_task_new_kfunc(meta) && + !check_css_task_iter_allowlist(env)) { + verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); + return -EINVAL; + } + err = process_iter_arg(env, reg, argno, insn_idx, meta); + if (err < 0) return err; break; + case ARG_PTR_TO_LIST_HEAD: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_ROOT: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_NODE: + if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) && + type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_NODE: + if (is_bpf_rbtree_add_kfunc(meta->func_id)) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } else { + if (!type_is_non_owning_ref(reg->type) && + !reg_is_referenced(env, reg)) { + verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", + meta->func_name); + return -EINVAL; + } + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "%s not allowed in rbtree cb\n", meta->func_name); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg, + "rdonly_buf_size")) + meta->r0_rdonly = true; + err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (err < 0) { + if (meta->btf && err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a verifier-known constant size for this function's buffer argument.", + "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); + return err; + } + break; + case ARG_PTR_TO_REFCOUNTED_KPTR: + { + struct btf_record *rec; + + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) + meta->arg_owning_ref = true; + + rec = reg_btf_record(reg); + if (!rec) { + verifier_bug(env, "Couldn't find btf_record"); + return -EFAULT; + } + + if (rec->refcount_off < 0) { + verbose(env, "%s doesn't point to a type with bpf_refcount field\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + break; + } case ARG_PTR_TO_CONST_STR: { err = check_arg_const_str(env, reg, argno); @@ -9100,6 +9406,38 @@ skip_type_check: return err; break; } + case ARG_PTR_TO_WORKQUEUE: + err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_TASK_WORK: + err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_IRQ_FLAG: + err = process_irq_flag(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RES_SPIN_LOCK: + { + int flags; + + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + + flags = get_bpf_res_spin_lock_kfunc_flags(meta); + if (!flags) + return -EFAULT; + err = process_spin_lock(env, reg, argno, flags); + if (err < 0) + return err; + break; + } case ARG_KPTR_XCHG_DEST: err = process_kptr_func(env, regno, meta); if (err) @@ -9110,6 +9448,37 @@ skip_type_check: return err; } +static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + const struct btf_param *args = NULL; + u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; + int err; + + if (meta->btf) { + args = btf_params(meta->func_proto); + nargs = btf_type_vlen(meta->func_proto); + } + + if (nargs > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, nargs, meta->func_name, + meta->btf, args); + if (err) + return err; + } + + for (arg = 0; arg < nargs; arg++) { + if (meta->fn->arg_type[arg] == ARG_UNUSED) + break; + err = check_func_arg(env, arg, meta, insn_idx); + if (err) + return err; + } + + return 0; +} + static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; @@ -11116,14 +11485,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn meta.func_id = func_id; meta.func_name = func_id_name(func_id); meta.fn = fn; - /* check args */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - if (fn->arg_type[i] == ARG_UNUSED) - break; - err = check_func_arg(env, i, &meta, insn_idx); - if (err) - return err; - } + err = check_func_args(env, &meta, insn_idx); + if (err) + return err; err = record_func_map(env, &meta, func_id, insn_idx); if (err) @@ -12092,7 +12456,10 @@ enum special_kfunc_type { KF_bpf_task_work_schedule_resume, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, + KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, + KF_bpf_stream_vprintk, + KF_bpf_stream_print_stack, }; BTF_ID_LIST(special_kfunc_list) @@ -12182,11 +12549,29 @@ BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) +BTF_ID(func, bpf_arena_reserve_pages) #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif +BTF_ID(func, bpf_stream_vprintk) +BTF_ID(func, bpf_stream_print_stack) + +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]); +} + +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]); +} + +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]); +} static bool is_bpf_obj_new_kfunc(u32 func_id) { @@ -12777,12 +13162,20 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta) { - return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; + int flags = PROCESS_RES_LOCK; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) + flags |= PROCESS_SPIN_LOCK; + else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) && + !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + return 0; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + flags |= PROCESS_LOCK_IRQ; + return flags; } static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) @@ -13059,449 +13452,6 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - int insn_idx) -{ - const char *func_name = meta->func_name; - struct bpf_func_state *caller = cur_func(env); - struct bpf_reg_state *regs = cur_regs(env); - const struct btf *btf = meta->btf; - const struct btf_param *args; - struct btf_record *rec; - u32 i, nargs; - int ret; - - args = (const struct btf_param *)(meta->func_proto + 1); - nargs = btf_type_vlen(meta->func_proto); - - ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); - if (ret) - return ret; - - /* Check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < nargs; i++) { - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - enum bpf_arg_type arg_type = meta->fn->arg_type[i]; - const struct btf_type *t; - argno_t argno = argno_from_arg(i + 1); - int regno = reg_from_argno(argno); - u32 ref_id = args[i].type; - u32 arg_size = arg_type & MEM_FIXED_SIZE ? meta->fn->arg_size[i] : 0; - - if (arg_type == ARG_PTR_TO_PROG_AUX) { - cur_aux(env)->arg_prog = regno; - continue; - } - - if (arg_type == ARG_IGNORE) - continue; - - t = btf_type_skip_modifiers(btf, args[i].type, NULL); - - if (btf_type_is_ptr(t)) - btf_type_skip_modifiers(btf, t->type, &ref_id); - - ret = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); - if (ret < 0) - return ret; - - ret = check_func_arg_release(env, reg, argno, arg_type, meta, insn_idx); - if (ret < 0) - return ret; - - if (reg_is_referenced(env, reg)) - update_ref_obj(&meta->ref_obj, reg); - - if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - ret = mark_arg_precision(env, argno); - if (ret) - return ret; - continue; - } - - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) - ref_id = *meta->fn->arg_btf_id[i]; - - ret = resolve_func_arg_type(env, reg, i, meta, &arg_type, &arg_size); - if (ret < 0) - return ret; - - ret = check_reg_type(env, reg, argno, arg_type, meta); - if (ret < 0) - return ret; - - ret = check_func_arg_reg_off(env, reg, argno, arg_type); - if (ret < 0) - return ret; - - switch (base_type(arg_type)) { - case ARG_CONST_SCALAR: - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case ARG_SCALAR: - break; - case ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) - meta->r0_rdonly = true; - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-known constant size for this kfunc buffer argument.", - "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case ARG_PTR_TO_CTX: - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { - ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); - if (ret < 0) - return -EINVAL; - meta->ret_btf_id = ret; - } - break; - case ARG_PTR_TO_ARENA: - break; - case ARG_PTR_TO_ALLOC_BTF_ID: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { - if (!is_bpf_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { - if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_percpu_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the owned object pointer before it is released or transferred.", - "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->btf == btf_vmlinux) { - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - } - break; - case ARG_PTR_TO_DYNPTR: - { - enum bpf_arg_type dynptr_arg_type = arg_type; - - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]) && - (dynptr_arg_type & MEM_UNINIT)) { - enum bpf_dynptr_type parent_type = meta->dynptr.type; - - if (parent_type == BPF_DYNPTR_TYPE_INVALID) { - verifier_bug(env, "no dynptr type for parent of clone"); - return -EFAULT; - } - - dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - } - - ret = process_dynptr_func(env, reg, argno, insn_idx, - dynptr_arg_type, meta); - if (ret < 0) - return ret; - break; - } - case ARG_PTR_TO_ITER: - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new])) { - if (!check_css_task_iter_allowlist(env)) { - verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); - return -EINVAL; - } - } - ret = process_iter_arg(env, reg, argno, insn_idx, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_LIST_HEAD: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_RB_ROOT: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_LIST_NODE: - if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && - type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { - /* Allow bpf_list_front/back return value for - * __nonown_allowed list-node arguments. - */ - goto check_ok; - } - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } -check_ok: - ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_RB_NODE: - if (is_bpf_rbtree_add_kfunc(meta->func_id)) { - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - } else { - if (!type_is_non_owning_ref(reg->type) && - !reg_is_referenced(env, reg)) { - verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); - return -EINVAL; - } - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "%s not allowed in rbtree cb\n", func_name); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_CONST_MAP_PTR: - ret = process_map_ptr_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type)) { - if (!(arg_type & MEM_RCU)) { - const char *actual_type, *arg_name, *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - arg_name = reg_arg_name(env, argno); - actual_type = bpf_diag_reg_type_plain(env, reg->type); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", - "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", - expected_type, arg_name, actual_type); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - const char *actual_type, *arg_name, *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - arg_name = reg_arg_name(env, argno); - actual_type = bpf_diag_reg_type_plain(env, reg->type); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Use this kfunc with a pointer that is valid in an RCU read lock region.", - "the kfunc requires an RCU-protected pointer to %s, but %s is %s", - expected_type, arg_name, actual_type); - return -EINVAL; - } - } - - ret = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, btf, - ref_id, meta, insn_idx); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_MEM: - if (arg_type & MEM_FIXED_SIZE) { - bool known_memory; - - ret = check_mem_reg(env, reg, argno, arg_size, BPF_READ | BPF_WRITE, - meta, &known_memory); - if (ret < 0) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - if (known_memory) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass memory with at least the required number of accessible bytes and suitable read and write access.", - "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", - arg_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", - "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", - arg_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return ret; - } - } - break; - case ARG_CONST_MEM_SIZE: - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - fallthrough; - case ARG_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); - struct bpf_reg_state *size_reg = reg; - argno_t buff_argno = argno_from_arg(i); - enum bpf_mem_size_failure failure; - - if (bpf_register_is_null(buff_reg)) - break; - - ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta, &failure); - if (ret < 0) { - const char *buff_arg, *size_arg; - - buff_arg = bpf_diag_arg_name(env, buff_argno); - size_arg = bpf_diag_arg_name(env, argno); - verbose(env, "%s and ", reg_arg_name(env, buff_argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, argno)); - if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { - bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, - "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", - "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", - size_arg, buff_arg); - } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { - if (reg_smin(size_reg) < 0) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may be negative because its signed minimum is %lld", - size_arg, reg_smin(size_reg)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", - size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); - } - return ret; - } - break; - } - case ARG_PTR_TO_FUNC: - meta->subprogno = reg->subprogno; - break; - case ARG_PTR_TO_REFCOUNTED_KPTR: - if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) - meta->arg_owning_ref = true; - - rec = reg_btf_record(reg); - if (!rec) { - verifier_bug(env, "Couldn't find btf_record"); - return -EFAULT; - } - - if (rec->refcount_off < 0) { - verbose(env, "%s doesn't point to a type with bpf_refcount field\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - break; - case ARG_PTR_TO_CONST_STR: - ret = check_arg_const_str(env, reg, argno); - if (ret) - return ret; - break; - case ARG_PTR_TO_WORKQUEUE: - ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_TIMER: - ret = process_timer_func(env, reg, argno, &meta->map); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_TASK_WORK: - ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_IRQ_FLAG: - ret = process_irq_flag(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case ARG_PTR_TO_RES_SPIN_LOCK: - { - int flags = PROCESS_RES_LOCK; - - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); - return -EACCES; - } - if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) - return -EFAULT; - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || - is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) - flags |= PROCESS_SPIN_LOCK; - if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || - is_kfunc_call(meta, - special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) - flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, reg, argno, flags); - if (ret < 0) - return ret; - break; - } - } - } - - return 0; -} - int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, @@ -13993,7 +13943,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_aux->non_sleepable = true; /* Check the arguments */ - err = check_kfunc_args(env, &meta, insn_idx); + err = check_func_args(env, &meta, insn_idx); if (err < 0) return err; diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index e415acc08b16..c32c6dab49bc 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -11,7 +11,7 @@ struct { const char *prog_name; const char *err_msg; } cb_refs_tests[] = { - { "underflow_prog", "release function bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, + { "underflow_prog", "R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_" }, { "leak_prog", "Unreleased reference id=4 alloc_insn=3" }, /* alloc_insn=3{2,3} */ { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index 731eaa09db96..ede6a17d7da3 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -191,7 +191,7 @@ int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired = (struct cgroup *)&path; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 0ca2e1a38a94..12c8ac6099ca 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -179,7 +179,7 @@ int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired = (struct task_struct *)&clone_flags; diff --git a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c index ff9ea5aa80ad..ec4e0f3ff792 100644 --- a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c +++ b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c @@ -42,7 +42,7 @@ int wakeup_source_access_lock_fields(void *ctx) } SEC("syscall") -__failure __msg("release function bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int wakeup_source_unlock_no_lock(void *ctx) { struct bpf_ws_lock *lock = (void *)0x1; From afe897e89918434be8600250dbdda977aa99f53d Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:01 -0700 Subject: [PATCH 121/660] bpf: Read a kfunc's __sz argument only when it is in a register Commit e0b7b91c72db ("bpf: Support stack arguments for kfunc calls") supported stack arguments for kfunc's. In bpf_kfunc_stack_access_bytes(), the size of a ptr + __sz pair is read from const_reg_vals[] at index 'BPF_REG_1 + arg + 1'. Past the fifth argument that index leaves the argument registers and reaches 6 through 9, which are the callee saved registers R6 through R9. The verifier does record constants for those, so a __sz argument passed on the stack can take the value of an unrelated register as its size. Fix it by guard size_reg which has to be less than or equal to MAX_BPF_FUNC_REG_ARGS. Fixes: e0b7b91c72db ("bpf: Support stack arguments for kfunc calls") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195201.982215-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 617a277c3558..0c6a404167ca 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13641,13 +13641,14 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - /* ptr + __sz/__szk pair: size is in the next register */ + /* ptr + __sz/__szk pair: the size follows the pointer */ if (arg + 1 < nargs && (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; - if (aux->const_reg_mask & BIT(size_reg)) { + if (size_reg <= MAX_BPF_FUNC_REG_ARGS && + (aux->const_reg_mask & BIT(size_reg))) { size = (s64)aux->const_reg_vals[size_reg]; goto out; } From 4196b1064bf26dd02665750b84ab91d2ad93a83b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:06 -0700 Subject: [PATCH 122/660] selftests/bpf: Add a test for an __int128 by-value argument A 128-bit integer is passed in two consecutive argument registers, but the verifier counts one argument register per parameter whatever its size. For __u64 take_i128_global(int a, u128 v, int c) the compiler passes a in R1, v in R2:R3 and c in R4, while the verifier marks only R1 through R3 at the entry of the global function. The callee then reads its own third parameter out of a register the verifier considers uninitialized, and the program is rejected for a register the source never names: Validating take_i128_global() func#1... 20: R1=scalar() R2=scalar() R3=scalar() R10=fp0 ; __noinline __u64 take_i128_global(int a, u128 v, int c) @ verifier_aggregate_arg.c:12 20: (bf) r0 = r2 ; R0=scalar(id=4) R2=scalar(id=4) ; return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; @ verifier_aggregate_arg.c:14 21: (bc) w1 = w1 ; R1=scalar(smin=0,smax=umax=0xffffffff,var_off=(0x0; 0xffffffff)) 22: (67) r1 <<= 32 ; R1=scalar(smax=0x7fffffff00000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000)) 23: (c7) r1 s>>= 32 ; R1=scalar(smin=0xffffffff80000000,smax=0x7fffffff) 24: (0f) r0 += r1 ; R0=scalar() R1=scalar(smin=0xffffffff80000000,smax=0x7fffffff) 25: (0f) r0 += r3 ; R0=scalar() R3=scalar() 26: (bc) w1 = w4 R4 !read_ok The log is from clang 23. LLVM 21/22 place the argument in the same registers. Add the test with the failure it produces now. A later patch, "bpf: Support __int128 as a by-value function argument", places the two slots and flips this test to __success. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195206.984633-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_aggregate_arg.c | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index b97381448248..7732df9bc870 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_aggregate_arg.skel.h" #include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" @@ -172,6 +173,7 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_verifier_aggregate_arg(void) { RUN_TESTS(verifier_aggregate_arg); } void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c new file mode 100644 index 000000000000..d90f754396d0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 u128; + +__noinline __u64 take_i128_global(int a, u128 v, int c) +{ + return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; +} + +SEC("tc") +/* + * The verifier counts one argument register for the __int128 and marks only + * R1 through R3 at the entry of take_i128_global(), while the compiler passed + * a in R1, v in R2:R3 and c in R4. + */ +__failure __msg("R4 !read_ok") +int aggregate_arg_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (take_i128_global(1, v, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +char _license[] SEC("license") = "GPL"; From 3bd9d96710f00b9e5a7218d1d1cbb93d29b0595b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:11 -0700 Subject: [PATCH 123/660] bpf: Rename bpf_subprog_info::arg_cnt to arg_slot_cnt Rename arg_cnt to arg_slot_cnt, as a later patch gives a parameter that takes two argument registers, an __int128 or a 16-byte aggregate, two slots. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195211.986081-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 6 +++--- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 19 +++++++++++-------- .../bpf/progs/verifier_stack_arg_order.c | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1e7593e8d5c5..120a32bdd451 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -828,7 +828,7 @@ struct bpf_subprog_info { bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -838,8 +838,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c693f2c345f2..9b0bd3898882 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8081,7 +8081,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); - sub->arg_cnt = nargs; + sub->arg_slot_cnt = nargs; if (nargs > MAX_BPF_FUNC_ARGS) { bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", MAX_BPF_FUNC_ARGS, tname, nargs); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c6a404167ca..cf2696d7b778 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10247,7 +10247,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), NULL, NULL); if (err) @@ -10259,7 +10259,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); args = btf_params(func_proto); - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -10267,7 +10267,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, /* check that BTF function arguments match actual types that the * verifier sees. */ - for (i = 0; i < sub->arg_cnt; i++) { + for (i = 0; i < sub->arg_slot_cnt; i++) { argno_t argno = argno_from_arg(i + 1); struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); struct bpf_subprog_arg_info *arg = &sub->args[i]; @@ -19647,13 +19647,14 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { + if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = BPF_REG_1; + i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; @@ -19696,7 +19697,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) goto out; } } - if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; @@ -19709,9 +19711,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { + if (ret || sub->arg_slot_cnt != 1 || + sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; - sub->arg_cnt = 1; + sub->arg_slot_cnt = 1; sub->stack_arg_cnt = 0; } } diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index 57f22691744a..ab1955852233 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -116,8 +116,8 @@ __naked void stack_arg_pruning_load_after_call(void) /* * "bad_ptr": the first arg is 'long *', which is not a recognized pointer * type for static subprogs (not ctx, dynptr, or tagged). btf_prepare_func_args() - * sets arg_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The subprog - * is marked unreliable but the call still proceeds for static subprogs. + * sets arg_slot_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The + * subprog is marked unreliable but the call still proceeds for static subprogs. */ __noinline __used __naked static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) From 3f1023cab8130e8d43271b57d36139273a6c48a0 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:16 -0700 Subject: [PATCH 124/660] bpf: Index global function arguments by argument slot btf_prepare_func_args() indexes sub->args[] by BTF parameter, so a parameter can only ever stand for a single argument register. Give the loop a second index: 'i' keeps walking the BTF parameters while 'slots_used' walks the argument slots, and sub->arg_slot_cnt becomes the number of slots, so that a later patch can give a parameter two of them. No functional change: every parameter still takes exactly one slot. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195216.987189-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9b0bd3898882..cdbae53bb94d 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8037,7 +8037,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; int err; - u32 i, nargs, btf_id; + u32 i, slots_used, nargs, btf_id; const char *tname; if (sub->args_cached) @@ -8122,8 +8122,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) /* Convert BTF function arguments into verifier types. * Only PTR_TO_CTX and SCALAR are supported atm. */ - for (i = 0; i < nargs; i++) { + for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; @@ -8145,7 +8146,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) btf_validate_prog_ctx_type(log, btf, t, i, prog_type, prog->expected_attach_type)) return -EINVAL; - sub->args[i].arg_type = ARG_PTR_TO_CTX; + sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; continue; } if (btf_is_dynptr_ptr(btf, t)) { @@ -8153,7 +8154,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; + sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { @@ -8168,10 +8169,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (kern_type_id < 0) return kern_type_id; - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; if (tags & ARG_TAG_NULLABLE) - sub->args[i].arg_type |= PTR_MAYBE_NULL; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_UNTRUSTED) { @@ -8185,8 +8187,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { - sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; - sub->args[i].mem_size = 0; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | + PTR_UNTRUSTED; + sub->args[slots_used].mem_size = 0; + slots_used++; continue; } @@ -8202,8 +8206,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) i, btf_type_str(ref_t), tname); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_ARENA) { @@ -8211,7 +8216,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_ARENA; + sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; continue; } if (is_global) { /* generic user data pointer */ @@ -8231,10 +8236,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; if (tags & ARG_TAG_NONNULL) - sub->args[i].arg_type &= ~PTR_MAYBE_NULL; - sub->args[i].mem_size = mem_size; + sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; + sub->args[slots_used].mem_size = mem_size; + slots_used++; continue; } @@ -8244,7 +8250,7 @@ skip_pointer: return -EINVAL; } if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_SCALAR; + sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } if (!is_global) @@ -8254,6 +8260,8 @@ skip_pointer: return -EINVAL; } + sub->arg_slot_cnt = slots_used; + sub->args_cached = true; return 0; From 4dd676ca5b7a3bdc18e217bdd618c5e73016aee4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:21 -0700 Subject: [PATCH 125/660] bpf: Support by-value struct arguments up to 16 bytes A global function taking a struct or union by value is rejected today: Arg#1 type STRUCT in tar() is not supported yet. Accept one of at most 16 bytes, which arrives in one or two consecutive argument registers. Only structs composed entirely of scalars are taken for now; btf_struct_is_composed_of() enforces that. The slots of a value are independent of each other, so the compiler may split one across the last argument register and the stack, as in static void f(int a, int b, int c, int d, struct pair p); or place it wholly past the registers, and the verifier describes either the same way. What has to follow the slots is stack_arg_cnt, recomputed from the slots consumed, together with the "no stack args in global functions" and JIT support checks, and the MAX_BPF_FUNC_ARGS bound, which now has to account for a parameter that takes two slots at once. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195221.987808-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 80 +++++++++++++++++++++++++++++++++++-------- kernel/bpf/verifier.c | 2 ++ 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index cdbae53bb94d..c560ee0c4347 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8018,6 +8018,29 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt return -EOPNOTSUPP; } +static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, + bool is_global, u32 slot_cnt, + struct bpf_subprog_info *sub) +{ + if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + if (is_global) { + bpf_log(log, + "global function %s() needs %d > %d argument slots, " + "stack args not supported\n", + tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d argument slots\n", + tname, slot_cnt); + return -EFAULT; + } + sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -8087,20 +8110,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!bpf_jit_supports_stack_args()) { - bpf_log(log, "JIT does not support function %s() with %d args\n", - tname, nargs); - return -EFAULT; - } - sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; - } - - if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", - tname, nargs, MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } + err = btf_check_arg_slots(log, tname, is_global, nargs, sub); + if (err) + return err; err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { @@ -8125,6 +8137,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + if (slots_used >= MAX_BPF_FUNC_ARGS) + goto too_many_slots; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; @@ -8253,6 +8268,33 @@ skip_pointer: sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } + if (btf_type_is_struct(t)) { + u32 nslots; + + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + if (!is_global) + return -EINVAL; + bpf_log(log, + "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " + "can be passed by value\n", + i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (!is_global) + return -EINVAL; + bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", + i, btf_type_str(t), tname); + return -EINVAL; + } + + nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + if (slots_used + nslots > MAX_BPF_FUNC_ARGS) + goto too_many_slots; + while (nslots--) + sub->args[slots_used++].arg_type = ARG_SCALAR; + continue; + } if (!is_global) return -EINVAL; bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", @@ -8260,11 +8302,21 @@ skip_pointer: return -EINVAL; } + err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); + if (err) + return err; sub->arg_slot_cnt = slots_used; sub->args_cached = true; return 0; + +too_many_slots: + if (!is_global) + return -EINVAL; + bpf_log(log, "Arguments of %s() need more than %d argument slots\n", + tname, MAX_BPF_FUNC_ARGS); + return -EINVAL; } static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cf2696d7b778..d8963d50d85b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10259,6 +10259,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); args = btf_params(func_proto); + if (sub->arg_slot_cnt != btf_type_vlen(func_proto)) + args = NULL; ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), btf, args); if (ret) From 00cfeb9554e2513cc721acdf5679bd26dcf24e02 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:27 -0700 Subject: [PATCH 126/660] bpf: Support __int128 as a by-value function argument A 128-bit integer follows the same calling convention as a 16-byte by-value struct: LLVM emits it as a 16-byte BTF_KIND_INT and passes it in two consecutive argument registers. So a __int128 gets its two slots. The test added at the start of the series, which recorded the wrong register map as an R4 !read_ok rejection, now passes and is flipped to __success. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195227.987991-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 9 +++------ .../testing/selftests/bpf/progs/verifier_aggregate_arg.c | 7 +------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c560ee0c4347..9f8a4e4aac3b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8264,11 +8264,7 @@ skip_pointer: bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); return -EINVAL; } - if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[slots_used++].arg_type = ARG_SCALAR; - continue; - } - if (btf_type_is_struct(t)) { + if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { u32 nslots; if (!t->size || t->size > 2 * BPF_REG_SIZE) { @@ -8280,7 +8276,8 @@ skip_pointer: i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); return -EINVAL; } - if (!btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (btf_type_is_struct(t) && + !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { if (!is_global) return -EINVAL; bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c index d90f754396d0..fc7c1b18bc40 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -17,12 +17,7 @@ __noinline __u64 take_i128_global(int a, u128 v, int c) } SEC("tc") -/* - * The verifier counts one argument register for the __int128 and marks only - * R1 through R3 at the entry of take_i128_global(), while the compiler passed - * a in R1, v in R2:R3 and c in R4. - */ -__failure __msg("R4 !read_ok") +__success __retval(0) int aggregate_arg_int128_c_test(struct __sk_buff *skb) { __u64 a = skb->len ^ MIX_A; From c6ff4e06033fede2fd1d41232ae942add649b759 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:32 -0700 Subject: [PATCH 127/660] bpf: Rename bpf_call_summary::num_params to arg_slot_cnt The field 'num_params' counts the argument registers and outgoing stack slots a helper or kfunc call takes. The next patch gives a 16-byte parameter two slots, so the name stops describing what the field holds. Rename 'num_params' to 'arg_slot_cnt'. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195232.988528-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/liveness.c | 10 +++++----- kernel/bpf/verifier.c | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 120a32bdd451..1b836c6d570f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1066,7 +1066,7 @@ static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) } struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 7165ea325961..44ecdc5b4ec2 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1434,21 +1434,21 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, num_params = 5; + int r, err, arg_slot_cnt = 5; if (bpf_pseudo_call(insn)) return 0; if (bpf_get_call_summary(env, insn, &cs)) - num_params = cs.num_params; + arg_slot_cnt = cs.arg_slot_cnt; - for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } - for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) { err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], r + MAX_BPF_FUNC_REG_ARGS, insn_idx); if (err) @@ -2199,7 +2199,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); def = mask_widen(def); use = mask_widen(use); break; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d8963d50d85b..8c784e9fd8b7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17843,11 +17843,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; - cs->num_params = 0; + cs->arg_slot_cnt = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { if (fn->arg_type[i] == ARG_UNUSED) break; - cs->num_params++; + cs->arg_slot_cnt++; } return true; } @@ -17859,7 +17859,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->num_params = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = btf_type_vlen(meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; @@ -17968,7 +17968,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ - clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); + clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; From ae6abae582b750418ae1b09d3b295c416c61e082 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:37 -0700 Subject: [PATCH 128/660] bpf: Recognize by-value struct and __int128 kfunc arguments A kfunc taking a struct or union by value is rejected today, and one taking an __int128 is accepted but mis-counted: Unrecognized R2 type STRUCT The kfunc arguments walk the same slot as a BPF-to-BPF call: one argument register per eightbyte, and a 16-byte value takes two. The outgoing stack argument count at the call site follows the slots for the same reason. Similar to BPF-to-BPF aggregate handling, a kfunc aggregate argument is only supported when it is composed of scalars. Everything that maps a kfunc argument to a register has to follow the slots too. An argument of a single eightbyte lands in the same place under every calling convention, so those are taken. The conventions the JIT has to reconcile do not agree on where a larger one goes, so refuse it for now with Function f arg#1 type INT cannot be passed at argument slot 1 on this architecture which the next patch turns into an answer from the JIT. The paths that handle a two-slot argument are therefore unreachable until then. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195237.988712-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 189 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 36 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8c784e9fd8b7..c33f1e1d1a1c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8238,7 +8238,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } static int resolve_func_arg_type(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 arg, + struct bpf_reg_state *reg, u32 arg, argno_t argno, struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size); static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -8885,7 +8885,7 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } -static int check_func_arg(struct bpf_verifier_env *env, u32 arg, +static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) { @@ -8893,8 +8893,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, const struct bpf_func_proto *fn = meta->fn; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); - argno_t argno = argno_from_arg(arg + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, arg); + argno_t argno = argno_from_arg(slot + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; int regno = reg_from_argno(argno); u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; @@ -8925,7 +8925,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return 0; } - err = resolve_func_arg_type(env, reg, arg, meta, &arg_type, &arg_size); + err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); if (err) return err; @@ -9220,8 +9220,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, case ARG_MEM_SIZE: case ARG_MEM_SIZE_OR_ZERO: { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, arg - 1); - argno_t buff_argno = argno_from_arg(arg); + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); + argno_t buff_argno = argno_from_arg(prev_slot + 1); enum bpf_mem_size_failure failure; const char *buff_arg, *size_arg; bool zero_size_allowed; @@ -9448,9 +9448,52 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } +/* + * The slots a parameter takes, from its BTF type. This has to agree with + * btf_func_model_arg_slots(), which answers the same from the size the + * func model recorded, or the verifier would check an argument at a slot + * the JIT does not place it at. + */ +static u32 kfunc_arg_slots(const struct btf_type *t) +{ + if (btf_type_is_int(t) || btf_type_is_struct(t)) + return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + return 1; +} + +static u32 kfunc_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; + + for (i = 0; i < nargs; i++) + slots_used += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + + return slots_used; +} + +/* The argument slot @slot holds an eightbyte of a by-value argument. */ +static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, + u32 slot, struct bpf_call_arg_meta *meta) +{ + struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); + argno_t argno = argno_from_arg(slot + 1); + int regno = reg_from_argno(argno); + int err; + + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + return check_reg_type(env, reg, argno, ARG_SCALAR, meta); +} + static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { + u32 slot = 0, prev_slot = 0, proto_slots, nslots; struct bpf_func_state *caller = cur_func(env); const struct btf_param *args = NULL; u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; @@ -9461,19 +9504,47 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met nargs = btf_type_vlen(meta->func_proto); } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - err = check_outgoing_stack_args(env, caller, nargs, meta->func_name, - meta->btf, args); + /* + * A by-value argument of more than one eightbyte takes a slot per + * eightbyte, so the slots a call occupies are no longer its parameter + * count. Only a proto whose parameters take a slot each can name the + * argument a stack slot belongs to. + */ + proto_slots = meta->btf ? kfunc_proto_slots(meta->btf, meta->func_proto) : nargs; + + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, + meta->btf, proto_slots == nargs ? args : NULL); if (err) return err; } - for (arg = 0; arg < nargs; arg++) { + for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { + const struct btf_type *t; + u32 k; + + nslots = 1; + if (args) { + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + nslots = kfunc_arg_slots(t); + } + if (meta->fn->arg_type[arg] == ARG_UNUSED) break; - err = check_func_arg(env, arg, meta, insn_idx); + err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); if (err) return err; + + /* + * check_func_arg() took the first slot. A parameter of more + * than one eightbyte is always a scalar, so every slot it + * takes beyond the first holds one too. + */ + for (k = 1; k < nslots; k++) { + err = check_arg_extra_slot(env, caller, slot + k, meta); + if (err) + return err; + } } return 0; @@ -12329,11 +12400,10 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, } static int resolve_func_arg_type(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 arg, + struct bpf_reg_state *reg, u32 arg, argno_t argno, struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size) { - argno_t argno = argno_from_arg(arg + 1); const struct btf_param *args; const struct btf_type *ref_t, *resolve_ret; const struct btf *btf; @@ -12661,12 +12731,12 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs, + const struct btf_param *args, int arg, int nargs, u32 slot, struct bpf_func_proto *proto) { const struct btf_type *t, *ref_t = NULL, *resolve_ret; const u32 *ref_id_ptr = NULL; - argno_t argno = argno_from_arg(arg + 1); + argno_t argno = argno_from_arg(slot + 1); const char *ref_tname = NULL; u32 ref_id, type_size; int arg_type; @@ -12695,6 +12765,23 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, return ARG_SCALAR; } + if (btf_type_is_struct(t)) { + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + verbose(env, + "%s type %s has size %u, only 1 to %d bytes " + "can be passed by value\n", + reg_arg_name(env, argno), btf_type_str(t), t->size, + 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_type_is_scalar_struct(env, meta->btf, t)) { + verbose(env, "%s type %s is not composed of scalars\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + return ARG_SCALAR; + } + if (!btf_type_is_ptr(t)) { verbose(env, "Unrecognized %s type %s\n", reg_arg_name(env, argno), btf_type_str(t)); @@ -12858,7 +12945,7 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg struct bpf_func_proto *proto) { const struct btf_param *args; - u32 i, nargs; + u32 i, nargs, slots_used; int arg_type; args = (const struct btf_param *)(meta->func_proto + 1); @@ -12868,20 +12955,41 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - meta->func_name, nargs); - return -ENOTSUPP; - } - for (i = 0; i < nargs; i++) { - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, proto); + for (i = 0, slots_used = 0; i < nargs; i++) { + const struct btf_type *t; + u32 nslots; + + t = btf_type_skip_modifiers(meta->btf, args[i].type, NULL); + nslots = kfunc_arg_slots(t); + /* + * The calling conventions the JIT has to reconcile do not + * agree on where an argument of more than one eightbyte goes, + * so refuse one until the JIT can say where this arch puts it. + */ + if (nslots > 1) { + verbose(env, + "Function %s arg#%d type %s cannot be passed at " + "argument slot %d on this architecture\n", + meta->func_name, i, btf_type_str(t), slots_used); + return -EINVAL; + } + slots_used += nslots; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, + slots_used - nslots, proto); if (arg_type < 0) return arg_type; proto->arg_type[i] = arg_type; } + if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", + meta->func_name, slots_used); + return -ENOTSUPP; + } + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } @@ -13621,7 +13729,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; - u32 nargs, type_size; + u32 i, slot, nargs, type_size; s64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) @@ -13630,23 +13738,32 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); - if (arg >= nargs) + + /* + * @arg is an argument slot and a 16-byte parameter takes two of them, + * so walk the parameters to find the one that starts at this slot. A + * slot holding the upper eightbyte of such a parameter belongs to no + * pointer, and neither does a slot past the last parameter. + */ + for (i = 0, slot = 0; i < nargs && slot < arg; i++) + slot += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + if (i >= nargs || slot != arg) return 0; - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) return 0; /* dynptr: fixed 16-byte on-stack representation */ - if (is_kfunc_arg_dynptr(btf, &args[arg])) { + if (is_kfunc_arg_dynptr(btf, &args[i])) { size = BPF_DYNPTR_SIZE; goto out; } /* ptr + __sz/__szk pair: the size follows the pointer */ - if (arg + 1 < nargs && - (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || - btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { + if (i + 1 < nargs && + (btf_param_match_suffix(btf, &args[i + 1], "__sz") || + btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; if (size_reg <= MAX_BPF_FUNC_REG_ARGS && @@ -13862,7 +13979,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id, ret_nregs = 1; + u32 i, proto_slots, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14288,11 +14405,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - nargs = btf_type_vlen(meta.func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + proto_slots = kfunc_proto_slots(desc_btf, meta.func_proto); + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; - u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; if (stack_arg_cnt > caller_info->stack_arg_cnt) @@ -17859,7 +17976,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->arg_slot_cnt = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = kfunc_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; From 1e6c598b02e72ddd62b4d4e0c05159794b8bddbf Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:42 -0700 Subject: [PATCH 129/660] bpf: Prepare kfunc arguments for the JIT from an ABI description The previous patch refuses a kfunc argument of more than one eightbyte. This patch allows up to 16 byte kfunc arguments. But different architectures have different ways to map the BPF calling convention (no gap, no backfill) to the native one. Rather than have each arch open-code where it wants an argument, describe the convention with a register count and four booleans, and let each arch set what applies to it: struct bpf_jit_arg_abi { u8 nr_arg_regs; bool even_reg_align; bool even_stack_align; bool split_at_boundary; bool backfill_after_stack; }; The four booleans are meant to cover x86-64, arm64, RISC-V LP64 and PowerPC64 ELFv2, although only x86-64 and arm64 fill the struct in here. bpf_jit_place_args() and bpf_jit_plan_arg_moves() use that description to work out where each argument belongs and which slots the JIT then has to move, to be used in the JIT later on. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195242.989416-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 15 ++++++ include/linux/bpf_verifier.h | 1 + include/linux/filter.h | 32 ++++++++++++ kernel/bpf/btf.c | 3 ++ kernel/bpf/core.c | 94 ++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 99 +++++++++++++++++++++++++++++++----- 6 files changed, 231 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d0066d744ceb..2a5fa346aada 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -993,6 +993,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1210,6 +1217,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1227,6 +1237,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1b836c6d570f..cf85141ea167 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1512,6 +1512,7 @@ enum btf_member_kind { bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/include/linux/filter.h b/include/linux/filter.h index 00ad8b63aa47..b17222db2efc 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1248,6 +1248,38 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9f8a4e4aac3b..7daf4c286c9b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7579,6 +7579,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf, { u8 flags = __get_type_fmodel_flags(t); + if (btf_func_arg_align(btf, t) > sizeof(u64)) + flags |= BTF_FMODEL_ALIGN16_ARG; + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; else if (btf_param_match_suffix(btf, arg, "__arena")) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c673b02d55a6..4e208cc94752 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,100 @@ bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) return false; } +/* + * How this arch places a by-value kfunc argument, or NULL for one that has + * not opted in and so only takes an argument of a single eightbyte, which + * every convention places in slot order. + */ +const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) +{ + return NULL; +} + +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot) +{ + u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; + bool on_stack = false; + + for (i = 0; i < fm->nr_args; i++) { + bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; + u32 pos; + + nslots = btf_func_model_arg_slots(fm, i); + + if (align16 && abi->even_reg_align) + nregs_used = round_up(nregs_used, 2); + + if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { + /* wholly in registers */ + pos = nregs_used; + nregs_used += nslots; + } else if (!on_stack && abi->split_at_boundary) { + /* the last registers hold what fits, the stack the rest */ + pos = nregs_used; + stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; + nregs_used = abi->nr_arg_regs; + on_stack = true; + } else { + /* wholly on the stack */ + if (align16 && abi->even_stack_align) + stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); + pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; + stack_off += nslots * BPF_REG_SIZE; + if (!abi->backfill_after_stack) + on_stack = true; + } + + for (k = 0; k < nslots; k++) + pos_of_slot[slot + k] = pos + k; + slot += nslots; + } + + return slot; +} + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves) +{ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 nslots, n = 0, s, back; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + back = nslots; + + /* + * An argument is two eightbytes at most, so it frees one register at + * most and only one argument ever moves down. Its destination is + * still in use, so carry it in the scratch. Only a lower slot can + * take the one it leaves, so the walk reaches it first. + */ + for (s = nslots; s > 0; s--) { + u8 slot = s - 1, pos = pos_of_slot[slot]; + + if (pos == slot) + continue; + + if (pos < slot) { + moves[n].dst = BPF_JIT_ARG_TMP; + back = slot; + } else { + moves[n].dst = pos; + } + moves[n].src = slot; + n++; + } + + if (back < nslots) { + moves[n].dst = pos_of_slot[back]; + moves[n].src = BPF_JIT_ARG_TMP; + n++; + } + + return n; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c33f1e1d1a1c..6c6b8d8520cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2844,7 +2844,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto); + const struct btf_func_model *fm, struct bpf_func_proto *proto); int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { @@ -2961,7 +2961,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc = &tab->descs[tab->nr_descs]; memset(desc, 0, sizeof(*desc)); - err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); if (err) return err; @@ -12729,6 +12729,58 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]); } +static u32 kfunc_abi_slots(const struct btf_func_model *fm) +{ + const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 i, nslots, slots = 0; + + for (i = 0; i < fm->nr_args; i++) + slots += btf_func_model_arg_slots(fm, i); + + if (!abi) + return slots; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + for (i = 0; i < nslots; i++) + if (pos_of_slot[i] + 1 > slots) + slots = pos_of_slot[i] + 1; + + return slots; +} + +static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) +{ + const struct btf_member *member; + const struct btf_type *mt; + u32 align, i; + + while (btf_type_is_array(t)) + t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); + + if (btf_type_is_int(t)) + return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; + if (!btf_type_is_struct(t)) + return BPF_REG_SIZE; + if (rec >= BTF_MEMBER_MAX_DEPTH) + return 0; + + for_each_member(i, t, member) { + mt = btf_type_skip_modifiers(btf, member->type, NULL); + align = __btf_func_arg_align(btf, mt, rec + 1); + if (!align) + return 0; + if (align > BPF_REG_SIZE) + return 2 * BPF_REG_SIZE; + } + return BPF_REG_SIZE; +} + +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) +{ + return __btf_func_arg_align(btf, t, 0); +} + static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const struct btf_param *args, int arg, int nargs, u32 slot, @@ -12942,9 +12994,12 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto) + const struct btf_func_model *fm, struct bpf_func_proto *proto) { + const struct bpf_jit_arg_abi *abi; + const struct btf *btf = meta->btf; const struct btf_param *args; + const struct btf_type *t; u32 i, nargs, slots_used; int arg_type; @@ -12957,17 +13012,35 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg } for (i = 0, slots_used = 0; i < nargs; i++) { - const struct btf_type *t; - u32 nslots; + u32 nslots = btf_func_model_arg_slots(fm, i); - t = btf_type_skip_modifiers(meta->btf, args[i].type, NULL); - nslots = kfunc_arg_slots(t); - /* - * The calling conventions the JIT has to reconcile do not - * agree on where an argument of more than one eightbyte goes, - * so refuse one until the JIT can say where this arch puts it. - */ if (nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (!btf_func_arg_align(btf, t)) { + verbose(env, + "Function %s arg#%d type %s nests structs more than " + "%d levels deep\n", + meta->func_name, i, btf_type_str(t), + BTF_MEMBER_MAX_DEPTH); + return -EINVAL; + } + } + slots_used += nslots; + } + + if (slots_used > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, + slots_used, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + + abi = bpf_jit_arg_abi(); + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (!abi && nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); verbose(env, "Function %s arg#%d type %s cannot be passed at " "argument slot %d on this architecture\n", @@ -14405,7 +14478,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - proto_slots = kfunc_proto_slots(desc_btf, meta.func_proto); + proto_slots = kfunc_abi_slots(&desc->func_model); if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; From a3195057406fbea09c784e8c97e86765989dee3b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:47 -0700 Subject: [PATCH 130/660] bpf, x86: Move kfunc arguments into the x86-64 calling convention The x86-64 allows 6 register arguments and then stack arguments, but backfilling the 6th register is possible, as in the following example: /* s is a 16-byte struct */ void kfunc(u64 a, u64 b, u64 c, u64 d, u64 e, struct big s, u64 f) BPF puts s in slots 5 and 6, R9 and the first stack slot, and f in slot 7, while x86-64 puts s in the two stack slots and f in R9. Emit the moves bpf_jit_plan_arg_moves() plans to bridge the two. Only one argument ever moves down, as an argument frees one register at most, so carrying that single value in the scratch register past its own destination is enough. In addition, the arena argument walk counts eightbytes rather than parameters, as an argument may take two registers. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195247.989857-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index bba351944202..2671e4118d00 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1839,6 +1839,65 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +static const struct bpf_jit_arg_abi x86_arg_abi = { + .nr_arg_regs = 6, + .backfill_after_stack = true, + .even_stack_align = true, +}; + +static const u8 x86_arg_reg[] = { + BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, +}; + +/* + * Move the arguments the x86-64 ABI places somewhere other than the argument + * slot the BPF calling convention gave them. @stack_base addresses the + * outgoing stack argument area from RBP. Return the number of emitted bytes. + */ +static int emit_kfunc_arg_moves(const struct btf_func_model *fm, s32 stack_base, u8 **pprog) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 nreg = x86_arg_abi.nr_arg_regs; + u8 *prog = *pprog, *start = prog; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&x86_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + bool dst_mem = dst != BPF_JIT_ARG_TMP && dst >= nreg; + bool src_mem = src != BPF_JIT_ARG_TMP && src >= nreg; + + /* + * Take the value into a register: the one it belongs in, the + * scratch when it is carried past its own destination, and + * BPF_REG_AX only to pass one stack slot to another. + */ + if (src == BPF_JIT_ARG_TMP) { + reg = AUX_REG; + } else if (src_mem) { + reg = dst == BPF_JIT_ARG_TMP ? AUX_REG : + dst_mem ? BPF_REG_AX : x86_arg_reg[dst]; + emit_ldx(&prog, BPF_DW, reg, BPF_REG_FP, + stack_base + (src - nreg) * 8); + } else { + reg = x86_arg_reg[src]; + } + + /* And leave it where the argument belongs. */ + if (dst == BPF_JIT_ARG_TMP) + emit_mov_reg(&prog, true, AUX_REG, reg); + else if (dst_mem) + emit_stx(&prog, BPF_DW, BPF_REG_FP, reg, + stack_base + (dst - nreg) * 8); + else if (reg != x86_arg_reg[dst]) + emit_mov_reg(&prog, true, x86_arg_reg[dst], reg); + } + + *pprog = prog; + return prog - start; +} + /* * Rebase the __arena args of a kfunc call to arena kernel addresses, * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable @@ -1850,11 +1909,17 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, { u8 *prog = *pprog; u8 *start = prog; - int i; + int i, slot; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; - u32 reg = BPF_REG_1 + i; + u32 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = BPF_REG_1 + slot; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -2837,6 +2902,8 @@ populate_extable: if (err < 0) return err; ip += err; + ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - + outgoing_rsp, &prog); } if (priv_frame_ptr) { push_r9(&prog); @@ -4351,6 +4418,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &x86_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; From afdbb4cdd3d2714c58c68aab131bd2a43b14feb4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:52 -0700 Subject: [PATCH 131/660] bpf, arm64: Place trampoline arguments by the arm64 calling convention calc_arg_aux(), save_args() and restore_args() number argument registers and stack slots consecutively, while AAPCS64 rounds both up to an even one for an argument aligned to 16 bytes: an __int128, or an aggregate holding one. The argument that takes the hole, and every argument after it, is saved from and restored to the wrong place, so a bpf program attached to such a function reads a neighbouring eightbyte, and the arguments handed on to the original function are shifted. Take the position of each argument slot from bpf_jit_place_args() and keep it in struct arg_aux. save_args() then reads a slot from the register, or the incoming stack slot, and restore_args() puts it back there; the outgoing area built for the original function mirrors the incoming one, hole and all. A bpf program still cannot read an __int128 argument itself: it is an integer wider than eight bytes, which btf_ctx_access() refuses, so the program is rejected whatever the trampoline does with the argument. What a program can read, and what the placement puts right, is an aggregate holding an __int128, allowed as a struct, and any argument that follows a 16-byte aligned one. Both were taken from the wrong place before. Bound the argument slots rather than the arguments while here. btf_distill_func_proto() only limits the count although a by-value argument may take two slots, so, similar to x86-64, support up to MAX_BPF_FUNC_ARGS argument slots and refuse a function with more. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195252.990578-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 78 ++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 3aa3ea0bc30b..25a7657a6710 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1220,6 +1220,12 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +static const struct bpf_jit_arg_abi arm64_arg_abi = { + .nr_arg_regs = 8, + .even_reg_align = true, + .even_stack_align = true, +}; + static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) }; #define NR_STACK_ARG_REGS ARRAY_SIZE(stack_arg_reg) @@ -1293,6 +1299,16 @@ static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *ins return 0; } +static bool a64_arg_on_stack(u8 slot) +{ + return slot >= arm64_arg_abi.nr_arg_regs; +} + +static s32 a64_arg_stack_off(u8 slot) +{ + return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -2529,33 +2545,41 @@ struct arg_aux { * arguments to be properly aligned) */ int ostack_for_args; + /* where AAPCS64 puts each argument slot: an argument register below + * the eighth, an on-stack argument slot from it up + */ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; }; static int calc_arg_aux(const struct btf_func_model *m, struct arg_aux *a) { - int stack_slots, nregs, slots, i; + int slots, i, slot, total; + + total = bpf_jit_place_args(&arm64_arg_abi, m, a->pos_of_slot); + if (total > MAX_BPF_FUNC_ARGS) + return -ENOTSUPP; /* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */ - for (i = 0, nregs = 0; i < m->nr_args; i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { slots = (m->arg_size[i] + 7) / 8; - if (nregs + slots <= 8) /* passed through register ? */ - nregs += slots; - else + if (a64_arg_on_stack(a->pos_of_slot[slot])) /* passed through register ? */ break; + slot += slots; } a->args_in_regs = i; - a->regs_for_args = nregs; + a->regs_for_args = slot; a->ostack_for_args = 0; a->bstack_for_args = 0; /* the rest arguments are passed through stack */ - for (; i < m->nr_args; i++) { - stack_slots = (m->arg_size[i] + 7) / 8; - a->bstack_for_args += stack_slots * 8; - a->ostack_for_args = a->ostack_for_args + stack_slots * 8; - } + for (; i < m->nr_args; i++) + a->bstack_for_args += ((m->arg_size[i] + 7) / 8) * 8; + + /* the outgoing area reaches the last slot, over any alignment hole */ + if (a->bstack_for_args) + a->ostack_for_args = a64_arg_stack_off(a->pos_of_slot[total - 1]) + 8; return 0; } @@ -2602,7 +2626,7 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, { u8 tmp = bpf2a64[TMP_REG_1]; u8 base_lo = bpf2a64[TMP_REG_2]; - int i, reg, doff, soff, slots; + int i, reg, slot, soff, slots; /* only the low 32 bits of the base take part in the subtraction */ if (arena_base) @@ -2611,12 +2635,13 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, /* store arguments to the stack for the bpf program, or restore * arguments from stack for the original function */ - for (i = 0, reg = 0; i < a->args_in_regs; i++) { + for (i = 0, slot = 0; i < a->args_in_regs; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; slots = (m->arg_size[i] + 7) / 8; while (slots-- > 0) { + reg = a->pos_of_slot[slot++]; if (for_call_origin) { emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); } else if (arena_arg) { @@ -2625,7 +2650,6 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, } else { emit(A64_STR64I(reg, A64_SP, bargs_off), ctx); } - reg++; bargs_off += 8; } } @@ -2637,9 +2661,11 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops * callback is called indirectly and only the FP/LR frame is saved, so * they start at FP + 16. + * + * The outgoing area mirrors the incoming one, hole and all; only the + * bpf program takes the arguments packed. */ soff = is_struct_ops ? 16 : 32; - doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ for (i = a->args_in_regs; i < m->nr_args; i++) { @@ -2649,7 +2675,9 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, slots = (m->arg_size[i] + 7) / 8; /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */ while (slots-- > 0) { - emit(A64_LDR64I(tmp, A64_FP, soff), ctx); + int off = a64_arg_stack_off(a->pos_of_slot[slot++]); + + emit(A64_LDR64I(tmp, A64_FP, soff + off), ctx); /* if there is unused space in the last slot, clear * the garbage contained in the space. */ @@ -2664,19 +2692,21 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, */ if (arena_arg) emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo); - emit(A64_STR64I(tmp, A64_SP, doff), ctx); - soff += 8; - doff += 8; + if (for_call_origin) + emit(A64_STR64I(tmp, A64_SP, oargs_off + off), ctx); + else + emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx); + bargs_off += 8; } } } -static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) +static void restore_args(struct jit_ctx *ctx, int bargs_off, const struct arg_aux *a) { - int reg; + int slot; - for (reg = 0; reg < nregs; reg++) { - emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); + for (slot = 0; slot < a->regs_for_args; slot++) { + emit(A64_LDR64I(a->pos_of_slot[slot], A64_SP, bargs_off), ctx); bargs_off += 8; } } @@ -2948,7 +2978,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, bargs_off, a->regs_for_args); + restore_args(ctx, bargs_off, a); /* restore callee saved register x19 and x20 */ emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); From e1891a2c64747fbe9ee236a6aa45e9203b02f99d Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:57 -0700 Subject: [PATCH 132/660] bpf, arm64: Move kfunc arguments into the arm64 calling convention Do the proper move from the BPF calling convention to the arm64 calling convention to satisfy the native requirement. AAPCS64 only ever moves an argument to a higher slot, so the moves need one scratch register to carry an eightbyte from one stack slot to another, and never the one a convention moving an argument down would need. In addition, the arena argument walk counts eightbytes rather than parameters, as an argument may take two registers. The walk takes the func model from the caller now, as the moves need it too, and runs first so that they carry the rebased value. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195257.990814-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 68 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 25a7657a6710..6c04fee46876 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1268,19 +1268,20 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested * on the truncated value as arena NULL is offset 0. */ -static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn) +static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct btf_func_model *fm) { const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; - const struct btf_func_model *fm; - int i; + int i, slot; - fm = bpf_jit_find_kfunc_model(ctx->prog, insn); - if (!fm) - return -EINVAL; - - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { - const u8 reg = bpf2a64[BPF_REG_1 + i]; + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; + u8 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = bpf2a64[BPF_REG_1 + slot]; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -1309,6 +1310,42 @@ static s32 a64_arg_stack_off(u8 slot) return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); } +/* + * Move the arguments AAPCS64 places somewhere other than the argument slot the + * BPF calling convention gave them. Slot N is X(N) up to the eighth, and the + * outgoing stack argument area from SP beyond it, both for the slot an + * argument comes from and for the one it goes to. + * + * AAPCS64 only ever moves an argument to a higher slot, so no move here ever + * takes BPF_JIT_ARG_TMP: bpf_jit_plan_arg_moves() hands out the scratch only + * for a convention that moves one down, which needs a register to carry the + * value past its own destination. + */ +static void emit_kfunc_arg_moves(struct jit_ctx *ctx, const struct btf_func_model *fm) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 tmp = bpf2a64[TMP_REG_1]; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&arm64_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + + if (a64_arg_on_stack(src)) { + reg = tmp; + emit(A64_LDR64I(reg, A64_SP, a64_arg_stack_off(src)), ctx); + } else { + reg = src; + } + + if (a64_arg_on_stack(dst)) + emit(A64_STR64I(reg, A64_SP, a64_arg_stack_off(dst)), ctx); + else if (reg != dst) + emit(A64_MOV(1, dst, reg), ctx); + } +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1732,9 +1769,15 @@ emit_cond_jmp: if (ret < 0) return ret; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - ret = emit_kfunc_arena_args(ctx, insn); + const struct btf_func_model *fm; + + fm = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!fm) + return -EINVAL; + ret = emit_kfunc_arena_args(ctx, fm); if (ret < 0) return ret; + emit_kfunc_arg_moves(ctx, fm); } emit_call(func_addr, ctx); /* @@ -2409,6 +2452,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &arm64_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; From 9b55be31d16d962bb5b1fa063a2a4ba780214743 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:02 -0700 Subject: [PATCH 133/660] selftests/bpf: Add C tests for by-value arguments up to 16 bytes Extend the by-value argument test with the aggregate cases, written in C so that they depend on the compiler lowering the argument into a pair of argument registers rather than on a hand-written register layout. The programs cover a struct and a union that fill two registers, a smaller struct that fills one, and two struct arguments in a row, alongside the __int128 already there. Each has an int argument around it so that a wrong slot count shows up as a wrong value in the parameters beside it; two pairs leave room for only one, which follows them. A global function taking a struct with a pointer member is rejected: the callee would receive the pointer as an opaque scalar. A struct the argument registers cannot hold reaches the callee partly on the stack, which the interpreter does not implement, so that case is loaded only when the JIT is on. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195302.991273-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../bpf/progs/verifier_aggregate_arg.c | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c index fc7c1b18bc40..b0ecccede47e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -7,6 +7,131 @@ #define MIX_A 0xdeadbeefcafef00dULL #define MIX_B 0x0123456789abcdefULL +struct pair { + __u64 lo; + __u64 hi; +}; + +struct small { + __u32 a; + __u32 b; +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; + __u64 hi; + } parts; +}; + +struct with_ptr { + void *p; + __u64 x; +}; + +static __noinline __u64 take_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_static_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +#if defined(__clang__) + +__noinline __u64 take_pair_global(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair_global(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_two_pairs_global(struct pair p, struct pair q, int d) +{ + return p.lo + p.hi + q.lo + q.hi + d; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_two_structs_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + struct pair q = { .lo = a + 1, .hi = b + 2 }; + + if (take_two_pairs_global(p, q, 3) != 2 * a + 2 * b + 6) + return 1; + + return 0; +} + +__noinline __u64 take_small_global(int a, struct small s, int c) +{ + return (__u64)a + s.a + s.b + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_small_struct_c_test(struct __sk_buff *skb) +{ + __u32 a = skb->len ^ (__u32)MIX_A; + __u32 b = skb->len ^ (__u32)MIX_B; + struct small s = { .a = a, .b = b }; + + if (take_small_global(1, s, 2) != (__u64)a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_upair_global(int a, union upair u, int c) +{ + return (__u64)a + u.parts.lo + u.parts.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair u; + + u.halves[0] = a; + u.halves[1] = b; + if (take_upair_global(1, u, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif + #ifdef __SIZEOF_INT128__ typedef unsigned __int128 u128; @@ -32,4 +157,44 @@ int aggregate_arg_int128_c_test(struct __sk_buff *skb) #endif /* __SIZEOF_INT128__ */ +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +static __noinline __u64 take_spilled_pair(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_spilled_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + int n = skb->len; + + if (take_spilled_pair(n, n + 1, n + 2, n + 3, p) != a + b + 4 * n + 6) + return 1; + + return 0; +} + +#endif + +__noinline __u64 take_with_ptr_global(struct with_ptr s) +{ + return s.x; +} + +SEC("tc") +__failure __msg("type STRUCT in take_with_ptr_global() is not composed of scalars") +int aggregate_arg_ptr_member_fail(struct __sk_buff *skb) +{ + struct with_ptr s = { .p = skb, .x = skb->len }; + + return take_with_ptr_global(s); +} + char _license[] SEC("license") = "GPL"; From a7a97c37c58fea0b0329b808d44dc7b0403db187 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:07 -0700 Subject: [PATCH 134/660] selftests/bpf: Add inline-asm tests for by-value arguments The tests cover a struct passed in a register pair, a pointer in one half of it refused at the call site, a struct too large to pass by value, and four placements a global function cannot have: six scalars, a struct split between the last argument register and the stack, one wholly past the registers, and an __int128 whose two slots push the last parameter out. The six-scalar case is the one whose slot count is known from the parameters alone, so it takes the check btf_prepare_func_args() makes before it walks them, while the other three take the one it makes after. Both report the same way. GCC passes an aggregate by invisible reference, so a callee it compiles expects a pointer where BTF says the halves of the struct are, and those tests are left to clang. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195308.992139-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_arg.c | 9 + .../selftests/bpf/progs/aggregate_arg_func.c | 188 ++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_arg.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_func.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c new file mode 100644 index 000000000000..b230f3bd3b2a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "aggregate_arg_func.skel.h" + +void test_aggregate_arg(void) +{ + RUN_TESTS(aggregate_arg_func); +} diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c new file mode 100644 index 000000000000..d0a4f84a6fbf --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct too_big { + __u64 a; + __u64 b; + __u64 c; +}; + +#if defined(__clang__) + +__noinline __u64 global_arg_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0x33) +__naked int aggregate_arg_pair_asm(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 0x10;" /* p.lo */ + "r3 = 0x20;" /* p.hi */ + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +SEC("tc") +__failure __msg("R2 is not a scalar") +__naked int aggregate_arg_pair_ptr_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = r10;" /* a stack pointer where p.lo belongs */ + "r3 = 0x20;" + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +#endif + +__noinline __u64 global_arg_too_big(struct too_big s) +{ + return s.a + s.b + s.c; +} + +SEC("tc") +__failure __msg("in global_arg_too_big() has size 24, only 1 to 16 bytes can be passed by value") +__naked int aggregate_arg_too_big_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "call %[global_arg_too_big];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_too_big) + : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +/* + * One slot per parameter, so the slot count is settled before the parameters + * are walked. The cases below reach the same count only once they have been. + */ +__noinline __u64 global_arg_six_scalars(int a, int b, int c, int d, int e, int f) +{ + return (__u64)a + b + c + d + e + f; +} + +SEC("tc") +__failure __msg("global function global_arg_six_scalars() needs 6 > 5 argument slots") +__naked int aggregate_arg_six_scalars_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_six_scalars];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_six_scalars) + : __clobber_all); +} + +__noinline __u64 global_arg_split(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_split() needs 6 > 5 argument slots") +__naked int aggregate_arg_split_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_split];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_split) + : __clobber_all); +} + +__noinline __u64 global_arg_past_regs(struct pair p, struct pair q, int a, struct pair r) +{ + return p.lo + p.hi + q.lo + q.hi + a + r.lo + r.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_past_regs() needs 7 > 5 argument slots") +__naked int aggregate_arg_past_regs_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_past_regs];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_past_regs) + : __clobber_all); +} + +#ifdef __SIZEOF_INT128__ + +__noinline __u64 global_arg_i128_slots(u128 v, int a, int b, int c, int d) +{ + return (__u64)v + a + b + c + d; +} + +SEC("tc") +__failure __msg("global function global_arg_i128_slots() needs 6 > 5 argument slots") +__naked int aggregate_arg_i128_slots_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_i128_slots];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_i128_slots) + : __clobber_all); +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +char _license[] SEC("license") = "GPL"; From 4006c4b06c2cb3c00c67688bd0da2cd3c95be210 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:13 -0700 Subject: [PATCH 135/660] selftests/bpf: Add tests for by-value kfunc arguments Add kfuncs taking a 16-byte struct and an __int128 by value, with combinations of <= 8 byte arguments and '> 8 && <= 16' byte arguments. Each kfunc weighs its parameters by argument slot, the first by one, the second by two and so on, and every test checks the value it returns. A plain sum would be the same whichever slot each value reached, so an argument that lands in the wrong one, or a 16-byte argument whose halves arrive the other way round, would pass quietly; a weighted one differs. An __int128 takes two argument slots. One test passes it in registers and one past them, where both conventions pad the stack to align it although the BPF convention does not, so both JITs move it up an eightbyte. Two more cover a rejection. An aggregate holding a pointer is refused everywhere, and in arena_kfunc.c a two-slot struct pushes an arena pointer past the argument registers, which is refused too. An aggregate too large to pass by value is already covered in aggregate_arg_func.c. test_stack_arg_big() in stack_arg_fail.c passed a 16-byte struct as the sixth argument and asserted the unrecognized stack argument type it used to be reported as. The JIT places that argument now, so the test is removed. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195313.992803-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_arg.c | 2 + .../selftests/bpf/progs/aggregate_arg_kfunc.c | 168 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_kfunc.c | 15 ++ .../selftests/bpf/progs/stack_arg_fail.c | 10 -- .../selftests/bpf/test_kmods/bpf_testmod.c | 59 +++++- .../bpf/test_kmods/bpf_testmod_kfunc.h | 27 +++ 6 files changed, 270 insertions(+), 11 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c index b230f3bd3b2a..aa7562f48737 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c @@ -2,8 +2,10 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include "aggregate_arg_func.skel.h" +#include "aggregate_arg_kfunc.skel.h" void test_aggregate_arg(void) { RUN_TESTS(aggregate_arg_func); + RUN_TESTS(aggregate_arg_kfunc); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c new file mode 100644 index 000000000000..52d3d5d53946 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#if defined(__clang__) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_struct(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg(1, s, 2) != 2 * a + 3 * b + 9) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg(1, 2, v) != 4 * a + 3 * b + 5) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_last_regs(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_nofit(1, 2, 3, 4, s) != 5 * a + 6 * b + 30) + return 1; + + return 0; +} + +/* + * The x86-64 ABI moves an argument its six remaining registers cannot hold + * wholly onto the stack. arm64, with eight argument registers, still has a + * pair for it. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_straddle(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_big_arg s = { .a = a, .b = b }; + + if (bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s) != 6 * a + 7 * b + 55) + return 1; + + return 0; +} + +/* The same, with an argument after the struct to take the eighth register. */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_tail(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_tail(1, 2, 3, 4, 5, s, 6) != 6 * a + 7 * b + 103) + return 1; + + return 0; +} + +/* + * arm64 gives no register to an argument its eight registers cannot hold, + * nor to anything after it. Past its six registers the x86-64 ABI has both + * eightbytes on the stack either way. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_split8(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_split8(1, 2, 3, 4, 5, 6, 7, s) != 8 * a + 9 * b + 140) + return 1; + + return 0; +} + +#ifdef __SIZEOF_INT128__ + +/* + * Both conventions pad the stack to align this __int128, and the BPF + * convention pads for neither, so both JITs move it up an eightbyte. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_pad(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_pad(1, 2, 3, 4, 5, 6, 7, v) != 9 * a + 8 * b + 140) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R1 type STRUCT is not composed of scalars") +int aggregate_arg_kfunc_ptr_member(struct __sk_buff *skb) +{ + struct prog_test_ptr_arg s = { .p = skb, .x = 1 }; + + return bpf_kfunc_call_test_ptr_arg(s); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 6578cf12fa27..96f7aacf4d9f 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -228,6 +228,21 @@ int arena_arg_stack(void *ctx) bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); return 0; } + +#if defined(__clang__) +/* The struct takes two slots, so the arena pointer is the sixth. */ +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack_after_pair(void *ctx) +{ + struct prog_test_pair_arg s = { .lo = 1, .hi = 2 }; + + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_call_test_pair_arena_arg(1, 2, 3, s, (u64 *)1); + return 0; +} +#endif #else SEC("syscall") __arch_x86_64 diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index eed97d582515..fff2e947ea33 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -3,20 +3,10 @@ #include #include -#include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" #if defined(__BPF_FEATURE_STACK_ARGUMENT) -SEC("tc") -__failure __msg("Unrecognized *(R11-8) type STRUCT") -int test_stack_arg_big(struct __sk_buff *skb) -{ - struct prog_test_big_arg s = { .a = 1, .b = 2 }; - - return bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s); -} - SEC("socket") __description("r11 in ALU instruction") __failure __msg("R11 is invalid") diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index f798bbbb4d13..66b14168566a 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1029,6 +1029,55 @@ __bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u6 return r; } +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg(u64 a, struct prog_test_pair_arg s, u64 b) +{ + return a + s.lo * 2 + s.hi * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) +{ + return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, __int128 v) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (u64)v * 8 + (u64)((unsigned __int128)v >> 64) * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_nofit(u64 a, u64 b, u64 c, u64 d, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + s.lo * 5 + s.hi * 6; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_tail(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, u64 f) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + f * 8; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_split8(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + s.lo * 8 + s.hi * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) +{ + return s.x; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arena_arg(u64 a, u64 b, u64 c, + struct prog_test_pair_arg s, + u64 *f__arena) +{ + return a + b + c + s.lo + s.hi + *f__arena; +} + __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) { struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; @@ -1186,7 +1235,7 @@ __bpf_kfunc u64 bpf_kfunc_call_stack_arg_timer(u64 a, u64 b, u64 c, u64 d, u64 e __bpf_kfunc u64 bpf_kfunc_call_stack_arg_big(u64 a, u64 b, u64 c, u64 d, u64 e, struct prog_test_big_arg s) { - return a + b + c + d + e + s.a + s.b; + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.a * 6 + s.b * 7; } static struct prog_test_ref_kfunc prog_test_struct = { @@ -1667,6 +1716,14 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_split8) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ptr_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arena_arg) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index b213ef14848b..d3696d5254c9 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -61,6 +61,16 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_pair_arg { /* 16 bytes: two argument registers */ + __u64 lo; + __u64 hi; +}; + +struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ + void *p; + __u64 x; +}; + struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ __u64 lo; __u64 hi; @@ -221,6 +231,23 @@ __int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +__u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; +#ifdef __SIZEOF_INT128__ +__u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __int128 v) __ksym; +#endif +__u64 bpf_kfunc_call_test_pair_arg_nofit(__u64 a, __u64 b, __u64 c, __u64 d, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_tail(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + struct prog_test_pair_arg s, __u64 f) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_split8(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arena_arg(__u64 a, __u64 b, __u64 c, + struct prog_test_pair_arg s, + __u64 *f__arena) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; From e18e82ba9ed53f60acce5f9995ee357a9f7ab91d Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Mon, 7 Sep 2026 17:05:36 +0000 Subject: [PATCH 136/660] libbpf: fix log level propagation for light skeleton loaders bpftool's -d option is documented to enable bpf_trace_printk() messages from the generated syscall loader when used with -L as specified in commit d510296d331a ("bpftool: Use syscall/loader program in "prog load" and "gen skeleton" command.") However commit b59e4ce8bcaa ("bpftool: Switch bpf_object__load_xattr() to bpf_object__load()") changed bpftool to use bpf_object__load() which call the internal bpf_object_load with extra_log_level to 0 instead of bpf_object__load_xattr with the user request log_level. All the plumbing was still there to generate the bpf_trace_printk() instructions from the generator but was now unreachable because bpf_gen__init() was called with extra_log_level to 0, leaving gen->log_level at 0. This uses the obj->log_level field introduced in commit e0e3ea888c69 ("libbpf: Allow passing user log setting through bpf_object_open_opts") set from reading verifier_logs in do_skeleton(). This preserves both object-level and explicit load-time logging settings. Fixes: b59e4ce8bcaa ("bpftool: Switch bpf_object__load_xattr() to bpf_object__load()") Acked-by: Daniel Borkmann Signed-off-by: Mahe Tardy Link: https://lore.kernel.org/r/20260907170536.9996-1-mahe.tardy@gmail.com Signed-off-by: Alexei Starovoitov --- tools/lib/bpf/libbpf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 8da5181c8e8b..5b0d87e1d0b2 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -9139,7 +9139,8 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch * permit cross-endian creation of "light skeleton". */ if (obj->gen_loader) { - bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); + bpf_gen__init(obj->gen_loader, obj->log_level | extra_log_level, + obj->nr_programs, obj->nr_maps); } else if (!is_native_endianness(obj)) { pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); return libbpf_err(-LIBBPF_ERRNO__ENDIAN); From 698181b2d5fda2192eeab4029e4e02646201d471 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 8 Sep 2026 15:35:06 +0800 Subject: [PATCH 137/660] selftests/bpf: Add test for indirect struct_ops trampoline Add a test case to verify that arguments passed on the stack are correctly read by an indirect struct_ops trampoline. This test ensures the correctness of stack offsets across various architectures. It is particularly critical for architectures like LoongArch, RISC-V, ARM64, and PowerPC where arguments beyond the first 8 registers are passed on the stack, as well as x86_64 and s390x which have lower register argument limits. Signed-off-by: Tiezhu Yang Link: https://lore.kernel.org/r/20260908073506.29734-1-yangtiezhu@loongson.cn Signed-off-by: Alexei Starovoitov --- .../prog_tests/test_struct_ops_multi_args.c | 57 ++++++++++++++++++- .../bpf/progs/struct_ops_multi_args.c | 14 ++++- .../selftests/bpf/test_kmods/bpf_testmod.c | 10 ++++ .../selftests/bpf/test_kmods/bpf_testmod.h | 5 ++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c index 0f321e889862..8767e5f8bcc7 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c @@ -3,7 +3,62 @@ #include #include "struct_ops_multi_args.skel.h" +static void test_refcounted_multi(void) +{ + struct struct_ops_multi_args *skel; + char log_buf[4096] = {}; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_log_buf(skel->progs.test_refcounted_multi, log_buf, sizeof(log_buf)); + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_EQ(err, -EINVAL, "struct_ops_multi_args__load")) + goto out; + + ASSERT_HAS_SUBSTR(log_buf, "program with __ref argument cannot tail call", + "check_verifier_log_message"); + +out: + struct_ops_multi_args__destroy(skel); +} + +static void test_trampoline_stack_args(void) +{ + struct struct_ops_multi_args *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_autoload(skel->progs.test_refcounted_multi, false); + skel->struct_ops.testmod_ref_acquire->test_refcounted_multi = NULL; + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_OK(err, "struct_ops_multi_args__load")) + goto out; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_ref_acquire); + if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) + goto out; + + ASSERT_EQ(skel->bss->trampoline_stack_arg9, 9999, "check_stack_passed_arg9"); + +out: + bpf_link__destroy(link); + struct_ops_multi_args__destroy(skel); +} + void test_struct_ops_multi_args(void) { - RUN_TESTS(struct_ops_multi_args); + if (test__start_subtest("test_refcounted_multi")) + test_refcounted_multi(); + + if (test__start_subtest("test_trampoline_stack_args")) + test_trampoline_stack_args(); } diff --git a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c index c62be15757f0..9d290f22b492 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c @@ -17,7 +17,6 @@ struct { } prog_array SEC(".maps"); SEC("struct_ops/test_refcounted_multi") -__failure __msg("program with __ref argument cannot tail call") int test_refcounted_multi(unsigned long long *ctx) { /* ctx[2] is used because the refcounted variable is the third argument */ @@ -29,7 +28,20 @@ int test_refcounted_multi(unsigned long long *ctx) return 0; } +__u64 trampoline_stack_arg9 = 0; + +SEC("struct_ops/test_trampoline_stack_args") +int BPF_PROG(test_trampoline_stack_args, int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + trampoline_stack_arg9 = arg9; + + return 0; +} + SEC(".struct_ops.link") struct bpf_testmod_ops testmod_ref_acquire = { .test_refcounted_multi = (void *)test_refcounted_multi, + .test_trampoline_stack_args = (void *)test_trampoline_stack_args, }; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 66b14168566a..fd2c0cdc91b1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1831,6 +1831,8 @@ static int bpf_dummy_reg(void *kdata, struct bpf_link *link) */ if (ops->test_2) ops->test_2(4, ops->data); + if (ops->test_trampoline_stack_args) + ops->test_trampoline_stack_args(1, 2, 3, 4, 5, 6, 7, 8, 9999); return 0; } @@ -1878,6 +1880,13 @@ bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref, return NULL; } +static int bpf_testmod_ops__test_trampoline_stack_args(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + return arg9; +} + static struct bpf_testmod_ops __bpf_testmod_ops = { .test_1 = bpf_testmod_test_1, .test_2 = bpf_testmod_test_2, @@ -1885,6 +1894,7 @@ static struct bpf_testmod_ops __bpf_testmod_ops = { .test_refcounted = bpf_testmod_ops__test_refcounted, .test_refcounted_multi = bpf_testmod_ops__test_refcounted_multi, .test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr, + .test_trampoline_stack_args = bpf_testmod_ops__test_trampoline_stack_args, }; struct bpf_struct_ops bpf_bpf_testmod_ops = { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 210b919290cc..2b12f81bf748 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -46,6 +46,11 @@ struct bpf_testmod_ops { struct task_struct *(*test_return_ref_kptr)(int dummy, struct task_struct *task, struct cgroup *cgrp); + /* Used to test indirect struct_ops trampoline with stack-passed arguments. */ + int (*test_trampoline_stack_args)(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9); + /* The following fields are used to test shadow copies. */ char onebyte; struct { From 29fcf9ce4bbdf4cc339def776860e1c0095bca98 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 10 Sep 2026 11:43:53 +0200 Subject: [PATCH 138/660] bpf, x86: Use global buffer for trampoline size generation Currently arch_bpf_trampoline_size allocates and frees a temporary trampoline buffer on every invocation. The buffer is only used as a scratch space while __arch_prepare_bpf_trampoline() calculates the required size, and the generated trampoline is discarded. Allocating a writable scratch page during kernel initialization and reusing it for all size calculations. This improves tracing_multi attachment time. With current code: # ./test_progs -t tracing_multi_bench_attach -v ... serial_test_tracing_multi_bench_attach: found 55227 functions serial_test_tracing_multi_bench_attach: attached in 1.563s serial_test_tracing_multi_bench_attach: detached in 0.256s With the fix: # ./test_progs -t tracing_multi_bench_attach -v ... serial_test_tracing_multi_bench_attach: found 55235 functions serial_test_tracing_multi_bench_attach: attached in 0.798s serial_test_tracing_multi_bench_attach: detached in 0.258s Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260910094353.1256316-1-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2671e4118d00..d4a980140b48 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,21 @@ void __asan_store8(void *p); static bool all_callee_regs_used[4] = {true, true, true, true}; +static void *trampoline_size_image; + +static int __init init_trampoline_size_image(void) +{ + /* + * The generated trampoline contains calls and jumps with 32bit relative + * offsets, so the scratch image must be in the execmem range. + * On x86, module data and executable memory share the same address range, + * so using EXECMEM_MODULE_DATA to get writable memory. + */ + trampoline_size_image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); + return trampoline_size_image ? 0 : -ENOMEM; +} +late_initcall(init_trampoline_size_image); + static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { if (len == 1) @@ -4067,24 +4083,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { struct bpf_tramp_image im; - void *image; - int ret; - /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). - * - * We cannot use kvmalloc here, because we need image to be in - * module memory range. - * Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA) - * that returns writable memory in the module address space. - */ - image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); - if (!image) + if (!trampoline_size_image) return -ENOMEM; - ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image, - m, flags, tnodes, func_addr); - execmem_free(image); - return ret; + return __arch_prepare_bpf_trampoline(&im, trampoline_size_image, + trampoline_size_image + PAGE_SIZE, + trampoline_size_image, m, flags, + tnodes, func_addr); } static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) From 3041e498f961faf734fe4a71525afb0f095cdff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Sat, 12 Sep 2026 10:41:00 +0200 Subject: [PATCH 139/660] libbpf: Add bpf_program__add_flags() and bpf_program__clear_flags() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When changing BPF program flags, applications often need to just add or remove a single flag, without touching the existing ones. When using the bpf_program__set_flags() function, this requires reading the existing flags and doing bitwise manipulations before resetting the result, which is slightly awkward to do, and easy to forget. Add two new convenience helpers, bpf_program__add_flags() and bpf_program__clear_flags(), which wraps bpf_program__set_flags() in the bitwise operations required to modify flags without clobbering the existing ones. Suggested-by: Andrii Nakryiko Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Reviewed-by: Jiayuan Chen Acked-by: Eduard Zingerman Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260912084109.432834-1-toke@redhat.com --- tools/lib/bpf/libbpf.c | 10 ++++++++++ tools/lib/bpf/libbpf.h | 22 ++++++++++++++++++++++ tools/lib/bpf/libbpf.map | 2 ++ 3 files changed, 34 insertions(+) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 5b0d87e1d0b2..613afae26519 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -9934,6 +9934,16 @@ int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) return 0; } +int bpf_program__add_flags(struct bpf_program *prog, __u32 flags) +{ + return bpf_program__set_flags(prog, prog->prog_flags | flags); +} + +int bpf_program__clear_flags(struct bpf_program *prog, __u32 flags) +{ + return bpf_program__set_flags(prog, prog->prog_flags & ~flags); +} + __u32 bpf_program__log_level(const struct bpf_program *prog) { return prog->log_level; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index b965ad571540..48266e752223 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -1011,6 +1011,28 @@ bpf_program__set_expected_attach_type(struct bpf_program *prog, LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_flags(struct bpf_program *prog, __u32 flags); +/** + * @brief **bpf_program__add_flags()** adds one or more flags to the BPF + * program, preserving any existing flags. + * + * @param prog BPF program + * @param flags the flags to add + * + * @return 0, on success; negative error code, otherwise + */ +LIBBPF_API int bpf_program__add_flags(struct bpf_program *prog, __u32 flags); + +/** + * @brief **bpf_program__clear_flags()** clears one or more flags from the BPF + * program, preserving any other flags not explicitly cleared. + * + * @param prog BPF program + * @param flags the flags to clear + * + * @return 0, on success; negative error code, otherwise + */ +LIBBPF_API int bpf_program__clear_flags(struct bpf_program *prog, __u32 flags); + /* Per-program log level and log buffer getters/setters. * See bpf_object_open_opts comments regarding log_level and log_buf * interactions. diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 08ab2ea881fb..a811a1b3a085 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -458,7 +458,9 @@ LIBBPF_1.7.0 { LIBBPF_1.8.0 { global: + bpf_program__add_flags; bpf_program__attach_tracing_multi; + bpf_program__clear_flags; bpf_program__clone; btf__find_by_name_kind_own; btf__new_empty_opts; From 13eee57365636d98e131b5177f4b023896a57bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Sat, 12 Sep 2026 10:41:01 +0200 Subject: [PATCH 140/660] selftests/bpf: Add assertions for bpf_program__{add,clear}_flags() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add assertions that round-tripping through bpf_program__add_flags() and bpf_program__clear_flags() ends up with the original flags value. Arbitrarily add these to the kernel_flag test prog since that's where we're already checking for the bpf_program__flags() value (and the test name contains "flag"). Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260912084109.432834-2-toke@redhat.com --- .../selftests/bpf/prog_tests/kernel_flag.c | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c index 25eb59f460ab..c723c9fbfbcb 100644 --- a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c +++ b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c @@ -10,15 +10,32 @@ void test_kernel_flag(void) struct test_kernel_flag *lsm_skel; struct kfunc_call_test *skel = NULL; struct kfunc_call_test_lskel *lskel = NULL; + struct bpf_program *prog; + __u32 flags; int ret; - lsm_skel = test_kernel_flag__open_and_load(); + lsm_skel = test_kernel_flag__open(); if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel")) return; - ASSERT_EQ(bpf_program__flags(lsm_skel->progs.bpf) & BPF_F_SLEEPABLE, + prog = lsm_skel->progs.bpf; + flags = bpf_program__flags(prog); + ASSERT_EQ(flags & BPF_F_SLEEPABLE, BPF_F_SLEEPABLE, "sleepable in program flags"); + ret = bpf_program__add_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__add_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags | BPF_F_ANY_ALIGNMENT, + "bpf_program__add_flags value"); + + ret = bpf_program__clear_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__clear_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags, "bpf_program__clear_flags value"); + + ret = test_kernel_flag__load(lsm_skel); + if (!ASSERT_OK(ret, "test_kernel_flag__load")) + goto close_prog; + lsm_skel->bss->monitored_tid = sys_gettid(); ret = test_kernel_flag__attach(lsm_skel); From 5f65b7160634be602390eb0cb942925e45a7f63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Sat, 12 Sep 2026 10:41:02 +0200 Subject: [PATCH 141/660] selftests/bpf: Adopt bpf_program__add_flags() helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt the newly added bpf_program__add_flags() helper everywhere the selftests adds program flags (replacing all uses of bpf_program__set_flags() in the tests). Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Reviewed-by: Jiayuan Chen Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260912084109.432834-3-toke@redhat.com --- tools/testing/selftests/bpf/prog_tests/attach_probe.c | 4 +--- tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c | 2 +- tools/testing/selftests/bpf/prog_tests/kasan.c | 4 ++-- tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c | 3 +-- tools/testing/selftests/bpf/prog_tests/xdp_metadata.c | 4 ++-- tools/testing/selftests/bpf/test_loader.c | 5 ++--- tools/testing/selftests/bpf/testing_helpers.c | 4 +--- tools/testing/selftests/bpf/veristat.c | 4 ++-- tools/testing/selftests/bpf/xdp_hw_metadata.c | 2 +- 9 files changed, 13 insertions(+), 19 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index 7dadb90e7b68..41136e1c7752 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -543,9 +543,7 @@ static void test_kprobe_sleepable(void) return; /* sleepable kprobe test case needs flags set before loading */ - if (!ASSERT_OK(bpf_program__set_flags( - skel->progs.handle_kprobe_sleepable, - bpf_program__flags(skel->progs.handle_kprobe_sleepable) | BPF_F_SLEEPABLE), + if (!ASSERT_OK(bpf_program__add_flags(skel->progs.handle_kprobe_sleepable, BPF_F_SLEEPABLE), "kprobe_sleepable_flags")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 652307a1b22e..902a70d5afeb 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, bpf_program__flags(prog) | testing_prog_flags()); + bpf_program__add_flags(prog, testing_prog_flags()); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c index 7cd4d1208c3b..9911b96fec9e 100644 --- a/tools/testing/selftests/bpf/prog_tests/kasan.c +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c @@ -424,13 +424,13 @@ void serial_test_kasan(void) prog = bpf_object__find_program_by_name(skel->obj, prog_name); if (!ASSERT_OK_PTR(prog, "find rnd_hi32 on_stack prog")) goto destroy; - bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", tests[i].prog_type, "not_on_stack"); prog = bpf_object__find_program_by_name(skel->obj, prog_name); if (!ASSERT_OK_PTR(prog, "find rnd_hi32 not_on_stack prog")) goto destroy; - bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); } if (!ASSERT_OK(kasan__load(skel), "load prog")) diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index ed3fd0a88dab..651123cd8602 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -361,8 +361,7 @@ static void test_attach_api_fails(void) sl_skel->bss->user_ptr = sl_skel; - err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, - bpf_program__flags(sl_skel->progs.handle_kprobe_multi_sleepable) | BPF_F_SLEEPABLE); + err = bpf_program__add_flags(sl_skel->progs.handle_kprobe_multi_sleepable, BPF_F_SLEEPABLE); if (!ASSERT_OK(err, "sleep_skel_set_flags")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 047dfdc322a2..3ab963458903 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -408,14 +408,14 @@ void test_xdp_metadata(void) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, rx_ifindex); - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); /* Make sure we can load a dev-bound program that performs * XDP_REDIRECT into a devmap. */ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect"); bpf_program__set_ifindex(new_prog, rx_ifindex); - bpf_program__set_flags(new_prog, bpf_program__flags(new_prog) | BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton")) goto out; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 794a7dfb0579..28724de06322 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -748,7 +748,7 @@ static void prepare_case(struct test_loader *tester, struct bpf_object *obj, struct bpf_program *prog) { - int min_log_level = 0, prog_flags; + int min_log_level = 0; if (env.verbosity > VERBOSE_NONE) min_log_level = 1; @@ -766,8 +766,7 @@ static void prepare_case(struct test_loader *tester, else bpf_program__set_log_level(prog, spec->log_level); - prog_flags = bpf_program__flags(prog); - bpf_program__set_flags(prog, prog_flags | spec->prog_flags); + bpf_program__add_flags(prog, spec->prog_flags); tester->log_buf[0] = '\0'; } diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 3f037949e978..d1d60451c5bc 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -292,7 +292,6 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, ); struct bpf_object *obj; struct bpf_program *prog; - __u32 flags; int err; obj = bpf_object__open_file(file, &opts); @@ -308,8 +307,7 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) bpf_program__set_type(prog, type); - flags = bpf_program__flags(prog) | testing_prog_flags(); - bpf_program__set_flags(prog, flags); + bpf_program__add_flags(prog, testing_prog_flags()); err = bpf_object__load(obj); if (err) diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e70741c6b9b7..9cfc9b4b41c1 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -1722,9 +1722,9 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf fixup_obj(obj, prog, base_filename); if (env.force_checkpoints) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); + bpf_program__add_flags(prog, BPF_F_TEST_STATE_FREQ); if (env.force_reg_invariants) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); + bpf_program__add_flags(prog, BPF_F_TEST_REG_INVARIANTS); opts.log_buf = buf; opts.log_size = buf_sz; diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index c5501b3fdf48..ab3f5fc56e86 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -845,7 +845,7 @@ int main(int argc, char *argv[]) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, ifindex); - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); printf("load bpf program...\n"); ret = xdp_hw_metadata__load(bpf_obj); From 5ef40d69b38a93bc9951dadb1a15c85c597e1a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Sat, 12 Sep 2026 10:41:03 +0200 Subject: [PATCH 142/660] bpftool: Adopt bpf_program__add_flags() helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt the newly added bpf_program__add_flags() helper to non-destructively add the BPF_F_XDP_DEV_BOUND_ONLY flag to XDP programs. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Acked-by: Quentin Monnet Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260912084109.432834-4-toke@redhat.com --- tools/bpf/bpftool/prog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 8c2f9255b36d..6ab911e82155 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1769,7 +1769,7 @@ offload_dev: } if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) { - bpf_program__set_flags(pos, bpf_program__flags(pos) | BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY); bpf_program__set_ifindex(pos, xdpmeta_ifindex); } else { bpf_program__set_ifindex(pos, offload_ifindex); From 10c4f610b215bf961235141161992f010cf7e451 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 16 Sep 2026 11:10:01 -0700 Subject: [PATCH 143/660] bpf: Avoid redundant IRQ save on bucket locks bpf_local_storage_update() takes the map bucket lock while holding local_storage->lock. bpf_selem_unlink_map() does the same; its only caller holds local_storage->lock. The outer lock is acquired with raw_res_spin_lock_irqsave(), so interrupts are already disabled at both sites. Using raw_res_spin_lock_irqsave() for the nested lock saves the already disabled IRQ state and issues another IRQ disable. The matching unlock tests that saved state before leaving interrupts disabled. On x86-64, this adds a pushfq/popq/cli sequence and a test/branch around an unreachable sti to each acquisition. Use raw_res_spin_lock() and raw_res_spin_unlock() instead. They retain preemption nesting, memory ordering and resilient-lock bookkeeping. The outer unlock remains responsible for restoring the caller's IRQ state. In the tested clang x86-64 build, this removes five executed instructions from each uncontended nested acquisition. It also shrinks bpf_local_storage_update() from 1732 to 1702 bytes and bpf_selem_unlink() from 1030 to 992 bytes. The affected paths are updates that add or replace an element in existing owner storage and successful unlinks. Document the owner-lock requirement of bpf_selem_unlink_map() and assert that interrupts are disabled. Signed-off-by: Usama Arif Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260916181001.201787-1-usama.arif@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/bpf_local_storage.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b5..4642da062f0f 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -240,24 +240,26 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, hlist_add_head_rcu(&selem->snode, &local_storage->list); } +/* Must be called with the owning local_storage->lock held. */ static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) { struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; struct bpf_local_storage_map_bucket *b; - unsigned long flags; int err; + lockdep_assert_irqs_disabled(); + local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held()); smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, flags); + err = raw_res_spin_lock(&b->lock); if (err) return err; hlist_del_init_rcu(&selem->map_node); - raw_res_spin_unlock_irqrestore(&b->lock, flags); + raw_res_spin_unlock(&b->lock); return 0; } @@ -552,7 +554,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, struct bpf_local_storage *local_storage; struct bpf_local_storage_map_bucket *b; HLIST_HEAD(old_selem_free_list); - unsigned long flags, b_flags; + unsigned long flags; int err; /* BPF_EXIST and BPF_NOEXIST cannot be both set */ @@ -637,7 +639,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, b_flags); + /* local_storage->lock is held, so IRQs are already disabled. */ + err = raw_res_spin_lock(&b->lock); if (err) goto unlock; @@ -655,7 +658,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, &old_selem_free_list); } - raw_res_spin_unlock_irqrestore(&b->lock, b_flags); + raw_res_spin_unlock(&b->lock); unlock: raw_res_spin_unlock_irqrestore(&local_storage->lock, flags); free_selem: From 71b75648d1a21c0a6379741d00f41086ec95f14a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 16 Sep 2026 23:24:07 +0000 Subject: [PATCH 144/660] selftests/bpf: Wait for links to come up in lwt_ip_encap lwt_ip_encap creates three fresh netns with veth pairs across them and then immediately pings over IPv6. The peer ends of the first veth pair end up with the same ifindex in their respective netns, so linkwatch does not treat the carrier-on as urgent and may deliver NETDEV_CHANGE up to one second later. Until that happens addrconf considers the link not ready: no ff00::/8 multicast route and no link-local address are installed, and the neighbour solicitation for the next hop is dropped as a no-route packet. The NS retransmit a second later succeeds, but ping -W1 has already given up: check_ping_ok:FAIL:ip netns exec ns-lwt-ip-encap-1-... ping -6 -c 1 -W1 -I veth1 fb04::1 > /dev/null unexpected error: 256 (errno 2) lwt_ip_encap:FAIL:ping OK unexpected error: -1 (errno 2) #226/1 lwt_ip_encap_ipv4/egress:FAIL This reproduces reliably when the test is run standalone on an idle system where setup completes well within a second. The original shell script had a "sleep 1 # reduce flakiness" after setup which was lost in the conversion to test_progs. Instead of sleeping, poll SIOCGIFFLAGS for IFF_RUNNING on all veths at the end of setup_network() so traffic is only sent once every link is operationally up. Fixes: f5e288943e2c ("selftests/bpf: Move test_lwt_ip_encap to test_progs") Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260916232407.764886-1-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/lwt_ip_encap.c | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 39e8a3b8b6af..5c5560d45c5b 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include +#include #include "network_helpers.h" #include "test_progs.h" @@ -141,6 +143,54 @@ fail: return 1; } +/* + * A veth whose peer sits in another netns with the same ifindex gets its + * carrier-on handled as a non-urgent linkwatch event, i.e. up to 1s late. + * Until then IPv6 considers the link not ready (no ff00::/8 route, no + * link-local address) and silently drops neighbour solicitations, so wait + * for all links to be operationally up (IFF_RUNNING) before sending traffic. + */ +static int wait_for_oper_up(const char *ns, const char *dev) +{ + struct nstoken *nstoken; + struct ifreq ifr = {}; + int i, fd, ret = -1; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "open ns")) + return -1; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_OK_FD(fd, "socket")) + goto out; + + strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1); + for (i = 0; i < 50; i++) { + if (!ASSERT_OK(ioctl(fd, SIOCGIFFLAGS, &ifr), "SIOCGIFFLAGS")) + break; + if (ifr.ifr_flags & IFF_RUNNING) { + ret = 0; + break; + } + usleep(100000); + } + close(fd); +out: + ASSERT_OK(ret, dev); + close_netns(nstoken); + return ret; +} + +static int wait_for_links(const char *ns1, const char *ns2, const char *ns3) +{ + if (wait_for_oper_up(ns1, "veth1") || wait_for_oper_up(ns1, "veth5") || + wait_for_oper_up(ns2, "veth2") || wait_for_oper_up(ns2, "veth3") || + wait_for_oper_up(ns2, "veth6") || wait_for_oper_up(ns2, "veth7") || + wait_for_oper_up(ns3, "veth4") || wait_for_oper_up(ns3, "veth8")) + return -1; + return 0; +} + static int configure_vrf(const char *ns1, const char *ns2) { if (!ns1 || !ns2) @@ -304,6 +354,9 @@ static int setup_network(char *ns1, char *ns2, char *ns3, const char *vrf) if (!ASSERT_OK(configure_ns3(ns3), "configure ns3 routes")) goto fail; + if (!ASSERT_OK(wait_for_links(ns1, ns2, ns3), "wait for links")) + goto fail; + /* Link bottom route to the GRE tunnels */ SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s %s", ns1, IP4_ADDR_GRE, IP4_ADDR_6, vrf); From b2cf39f53478fd4b04d2f04585c21b3d91d7042f Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 16 Sep 2026 12:26:40 -0700 Subject: [PATCH 145/660] bpftool: Accept zero perf counter snapshots A perf counter can be zero at fentry when PMU multiplexing has not scheduled the event. Do not reject that sample. Use enabled time to distinguish a successful snapshot. Read directly into the per-CPU map value. Signed-off-by: Mykyta Yatsenko Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260916-bpftool_cyles_per_run-v4-1-f2de02c480d5@meta.com --- tools/bpf/bpftool/skeleton/profiler.bpf.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tools/bpf/bpftool/skeleton/profiler.bpf.c b/tools/bpf/bpftool/skeleton/profiler.bpf.c index f48c783cb9f7..6685e7252fb5 100644 --- a/tools/bpf/bpftool/skeleton/profiler.bpf.c +++ b/tools/bpf/bpftool/skeleton/profiler.bpf.c @@ -45,28 +45,21 @@ const volatile __u32 num_metric = 1; SEC("fentry/XXX") int BPF_PROG(fentry_XXX) { - struct bpf_perf_event_value___local *ptrs[MAX_NUM_METRICS]; u32 key = bpf_get_smp_processor_id(); u32 i; - /* look up before reading, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { + struct bpf_perf_event_value___local *reading; u32 flag = i; - - ptrs[i] = bpf_map_lookup_elem(&fentry_readings, &flag); - if (!ptrs[i]) - return 0; - } - - for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { - struct bpf_perf_event_value___local reading; int err; - err = bpf_perf_event_read_value(&events, key, (void *)&reading, - sizeof(reading)); + reading = bpf_map_lookup_elem(&fentry_readings, &flag); + if (!reading) + return 0; + err = bpf_perf_event_read_value(&events, key, (void *)reading, + sizeof(*reading)); if (err) return 0; - *(ptrs[i]) = reading; key += num_cpu; } @@ -80,7 +73,7 @@ fexit_update_maps(u32 id, struct bpf_perf_event_value___local *after) before = bpf_map_lookup_elem(&fentry_readings, &id); /* only account samples with a valid fentry_reading */ - if (before && before->counter) { + if (before && before->enabled) { struct bpf_perf_event_value___local *accum; diff.counter = after->counter - before->counter; From 6b5fe4b6d5ae5fac328416b061e5416d68aec762 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 16 Sep 2026 12:26:41 -0700 Subject: [PATCH 146/660] bpftool: Group profile events by CPU Independently scheduled perf events can cover different intervals, making derived ratios inconsistent. Open one event group per CPU and enable it only after all selected metrics have joined. Fail the profile setup when a selected event cannot join its group. Signed-off-by: Mykyta Yatsenko Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260916-bpftool_cyles_per_run-v4-2-f2de02c480d5@meta.com --- tools/bpf/bpftool/prog.c | 89 ++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 6ab911e82155..a14a1a9601bd 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -2350,7 +2350,7 @@ static char *profile_tgt_name; static int *profile_perf_events; static int profile_perf_event_cnt; -static void profile_close_perf_events(struct profiler_bpf *obj) +static void profile_close_perf_events(void) { int i; @@ -2361,61 +2361,78 @@ static void profile_close_perf_events(struct profiler_bpf *obj) profile_perf_event_cnt = 0; } -static int profile_open_perf_event(int mid, int cpu, int map_fd) +static int profile_open_perf_event(int mid, int cpu, + int map_fd, int group_fd, __u32 map_key) { + struct perf_event_attr attr = metrics[mid].attr; + bool group_leader = group_fd < 0; int pmu_fd; + int err; - pmu_fd = syscall(__NR_perf_event_open, &metrics[mid].attr, - -1 /*pid*/, cpu, -1 /*group_fd*/, 0); + attr.disabled = group_leader; + pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, cpu, + group_fd, 0); if (pmu_fd < 0) { - if (errno == ENODEV) { - p_info("cpu %d may be offline, skip %s profiling.", - cpu, metrics[mid].name); - profile_perf_event_cnt++; - return 0; - } - return -1; + err = -errno; + if (errno == ENODEV && group_leader) + p_info("cpu %d may be offline, skip profiling.", cpu); + return err; } - if (bpf_map_update_elem(map_fd, - &profile_perf_event_cnt, - &pmu_fd, BPF_ANY) || - ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) { + if (bpf_map_update_elem(map_fd, &map_key, &pmu_fd, BPF_ANY)) { + err = -errno; close(pmu_fd); - return -1; + return err; } profile_perf_events[profile_perf_event_cnt++] = pmu_fd; - return 0; + return pmu_fd; } static int profile_open_perf_events(struct profiler_bpf *obj) { + __u32 num_cpu = obj->rodata->num_cpu; + __u32 map_key; unsigned int cpu, m; - int map_fd; + int group_fd, map_fd, pmu_fd; + int err; - profile_perf_events = calloc( - obj->rodata->num_cpu * obj->rodata->num_metric, sizeof(int)); + profile_perf_events = calloc(num_cpu * obj->rodata->num_metric, + sizeof(*profile_perf_events)); if (!profile_perf_events) { p_err("failed to allocate memory for perf_event array: %s", strerror(errno)); - return -1; - } - map_fd = bpf_map__fd(obj->maps.events); - if (map_fd < 0) { - p_err("failed to get fd for events map"); - return -1; + return -ENOMEM; } - for (m = 0; m < ARRAY_SIZE(metrics); m++) { - if (!metrics[m].selected) - continue; - for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) { - if (profile_open_perf_event(m, cpu, map_fd)) { - p_err("failed to create event %s on cpu %u", - metrics[m].name, cpu); - return -1; + map_fd = bpf_map__fd(obj->maps.events); + for (cpu = 0; cpu < num_cpu; cpu++) { + group_fd = -1; + map_key = cpu; + for (m = 0; m < ARRAY_SIZE(metrics); m++) { + if (!metrics[m].selected) + continue; + + pmu_fd = profile_open_perf_event(m, cpu, map_fd, group_fd, + map_key); + if (pmu_fd == -ENODEV && group_fd < 0) + break; + if (pmu_fd < 0) { + p_err("failed to add event %s to group on CPU %u: %s", + metrics[m].name, cpu, strerror(-pmu_fd)); + return pmu_fd; } + if (group_fd < 0) + group_fd = pmu_fd; + map_key += num_cpu; + } + if (group_fd < 0) + continue; + if (ioctl(group_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP)) { + err = -errno; + p_err("failed to enable perf event group on CPU %u: %s", + cpu, strerror(-err)); + return err; } } return 0; @@ -2423,7 +2440,7 @@ static int profile_open_perf_events(struct profiler_bpf *obj) static void profile_print_and_cleanup(void) { - profile_close_perf_events(profile_obj); + profile_close_perf_events(); profile_read_values(profile_obj); profile_print_readings(); profiler_bpf__destroy(profile_obj); @@ -2529,7 +2546,7 @@ static int do_profile(int argc, char **argv) return 0; out: - profile_close_perf_events(profile_obj); + profile_close_perf_events(); if (profile_obj) profiler_bpf__destroy(profile_obj); close(profile_tgt_fd); From c2c8bd265627364a8257c413e697fee8f2605197 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 16 Sep 2026 12:26:42 -0700 Subject: [PATCH 147/660] bpftool: Scale counters and report cycles per run Perf counters undercount when the PMU multiplexes them. Scale each per-CPU value before aggregation. Per-CPU event groups make PMU ratios use matching scheduling intervals. Keep run_cnt as the total number of program executions, independent of counter scheduling. Keep JSON value raw, report the scaled value in value_scaled, and document the output. Signed-off-by: Mykyta Yatsenko Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260916-bpftool_cyles_per_run-v4-3-f2de02c480d5@meta.com --- .../bpftool/Documentation/bpftool-prog.rst | 11 +- tools/bpf/bpftool/prog.c | 122 ++++++++++++------ 2 files changed, 89 insertions(+), 44 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 90fa2a48cc26..90fe8c61bf42 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -217,7 +217,14 @@ bpftool prog run *PROG* data_in *FILE* [data_out *FILE* [data_size_out *L*]] [ct bpftool prog profile *PROG* [duration *DURATION*] *METRICs* Profile *METRICs* for bpf program *PROG* for *DURATION* seconds or until user hits . *DURATION* is optional. If *DURATION* is not specified, - the profiling will run up to **UINT_MAX** seconds. + the profiling will run up to **UINT_MAX** seconds. Selected metrics form a + perf event group on each CPU so that ratios use counters scheduled over the + same intervals. Plain output scales each per-CPU metric value to correct + for perf event multiplexing. When **cycles** is selected, it also reports + cycles per program run. + + In JSON output, **value** is raw, **value_scaled** is scaled, and + **run_cnt** is the total number of program runs. bpftool prog help Print short help message. @@ -360,7 +367,7 @@ EXAMPLES :: 51397 run_cnt - 40176203 cycles (83.05%) + 40176203 cycles # 781.68 cycles per run (83.05%) 42518139 instructions # 1.06 insns per cycle (83.39%) 123 llc_misses # 2.89 LLC misses per million insns (83.15%) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a14a1a9601bd..000774835b10 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -2062,37 +2062,52 @@ static int do_profile(int argc, char **argv) #include "profiler.skel.h" +enum ratio_metric { + METRIC_NONE = -2, + METRIC_RUN_CNT = -1, + METRIC_CYCLES = 0, + METRIC_INSTRUCTIONS = 1, + METRIC_L1D_LOADS = 2, + METRIC_LLC_MISSES = 3, + METRIC_ITLB_MISSES = 4, + METRIC_DTLB_MISSES = 5, +}; + struct profile_metric { const char *name; struct bpf_perf_event_value val; + __u64 scaled_val; struct perf_event_attr attr; bool selected; /* calculate ratios like instructions per cycle */ - const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */ + const enum ratio_metric ratio_metric; const char *ratio_desc; const float ratio_mul; } metrics[] = { - { + [METRIC_CYCLES] = { .name = "cycles", .attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES, .exclude_user = 1, }, + .ratio_metric = METRIC_RUN_CNT, + .ratio_desc = "cycles per run", + .ratio_mul = 1.0, }, - { + [METRIC_INSTRUCTIONS] = { .name = "instructions", .attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS, .exclude_user = 1, }, - .ratio_metric = 1, + .ratio_metric = METRIC_CYCLES, .ratio_desc = "insns per cycle", .ratio_mul = 1.0, }, - { + [METRIC_L1D_LOADS] = { .name = "l1d_loads", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2102,8 +2117,9 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), .exclude_user = 1, }, + .ratio_metric = METRIC_NONE, }, - { + [METRIC_LLC_MISSES] = { .name = "llc_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2113,11 +2129,11 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "LLC misses per million insns", .ratio_mul = 1e6, }, - { + [METRIC_ITLB_MISSES] = { .name = "itlb_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2127,11 +2143,11 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "itlb misses per million insns", .ratio_mul = 1e6, }, - { + [METRIC_DTLB_MISSES] = { .name = "dtlb_misses", .attr = { .type = PERF_TYPE_HW_CACHE, @@ -2141,7 +2157,7 @@ struct profile_metric { (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), .exclude_user = 1 }, - .ratio_metric = 2, + .ratio_metric = METRIC_INSTRUCTIONS, .ratio_desc = "dtlb misses per million insns", .ratio_mul = 1e6, }, @@ -2182,7 +2198,7 @@ static int profile_parse_metrics(int argc, char **argv) return selected_cnt; } -static void profile_read_values(struct profiler_bpf *obj) +static int profile_read_values(struct profiler_bpf *obj) { __u32 m, cpu, num_cpu = obj->rodata->num_cpu; int reading_map_fd, count_map_fd; @@ -2192,15 +2208,11 @@ static void profile_read_values(struct profiler_bpf *obj) reading_map_fd = bpf_map__fd(obj->maps.accum_readings); count_map_fd = bpf_map__fd(obj->maps.counts); - if (reading_map_fd < 0 || count_map_fd < 0) { - p_err("failed to get fd for map"); - return; - } err = bpf_map_lookup_elem(count_map_fd, &key, counts); if (err) { p_err("failed to read count_map: %s", strerror(errno)); - return; + return err; } profile_total_count = 0; @@ -2208,24 +2220,37 @@ static void profile_read_values(struct profiler_bpf *obj) profile_total_count += counts[cpu]; for (m = 0; m < ARRAY_SIZE(metrics); m++) { - struct bpf_perf_event_value values[num_cpu]; + struct bpf_perf_event_value values[num_cpu], *val; + double scale; if (!metrics[m].selected) continue; err = bpf_map_lookup_elem(reading_map_fd, &key, values); if (err) { - p_err("failed to read reading_map: %s", - strerror(errno)); - return; + p_err("failed to read reading_map: %s", strerror(errno)); + return err; } + for (cpu = 0; cpu < num_cpu; cpu++) { - metrics[m].val.counter += values[cpu].counter; - metrics[m].val.enabled += values[cpu].enabled; - metrics[m].val.running += values[cpu].running; + val = &values[cpu]; + if (counts[cpu] && !val->running) { + p_err("perf event %s was not counted on CPU %u", + metrics[m].name, cpu); + return -EAGAIN; + } + metrics[m].val.enabled += val->enabled; + metrics[m].val.running += val->running; + metrics[m].val.counter += val->counter; + if (val->running) { + /* Scale values to account for perf event multiplexing. */ + scale = (double)val->enabled / val->running; + metrics[m].scaled_val += val->counter * scale; + } } key++; } + return 0; } static void profile_print_readings_json(void) @@ -2242,6 +2267,7 @@ static void profile_print_readings_json(void) jsonw_lluint_field(json_wtr, "value", metrics[m].val.counter); jsonw_lluint_field(json_wtr, "enabled", metrics[m].val.enabled); jsonw_lluint_field(json_wtr, "running", metrics[m].val.running); + jsonw_lluint_field(json_wtr, "value_scaled", metrics[m].scaled_val); jsonw_end_object(json_wtr); } @@ -2250,24 +2276,34 @@ static void profile_print_readings_json(void) static void profile_print_readings_plain(void) { - __u32 m; + __u32 i; printf("\n%18llu %-20s\n", profile_total_count, "run_cnt"); - for (m = 0; m < ARRAY_SIZE(metrics); m++) { - struct bpf_perf_event_value *val = &metrics[m].val; + for (i = 0; i < ARRAY_SIZE(metrics); i++) { + struct profile_metric *m = &metrics[i]; + struct bpf_perf_event_value *val = &m->val; int r; + __u64 ratio; - if (!metrics[m].selected) + if (!m->selected) continue; - printf("%18llu %-20s", val->counter, metrics[m].name); + printf("%18llu %-20s", m->scaled_val, m->name); - r = metrics[m].ratio_metric - 1; - if (r >= 0 && metrics[r].selected && - metrics[r].val.counter > 0) { + r = m->ratio_metric; + switch (r) { + case METRIC_RUN_CNT: + ratio = profile_total_count; + break; + case METRIC_NONE: + ratio = 0; + break; + default: + ratio = metrics[r].scaled_val; + } + if (ratio) { printf("# %8.2f %-30s", - val->counter * metrics[m].ratio_mul / - metrics[r].val.counter, - metrics[m].ratio_desc); + m->scaled_val * m->ratio_mul / ratio, + m->ratio_desc); } else { printf("%-41s", ""); } @@ -2438,21 +2474,24 @@ static int profile_open_perf_events(struct profiler_bpf *obj) return 0; } -static void profile_print_and_cleanup(void) +static int profile_print_and_cleanup(void) { + int err; + profile_close_perf_events(); - profile_read_values(profile_obj); - profile_print_readings(); + err = profile_read_values(profile_obj); + if (!err) + profile_print_readings(); profiler_bpf__destroy(profile_obj); close(profile_tgt_fd); free(profile_tgt_name); + return err; } static void int_exit(int signo) { - profile_print_and_cleanup(); - exit(0); + exit(!!profile_print_and_cleanup()); } static int do_profile(int argc, char **argv) @@ -2542,8 +2581,7 @@ static int do_profile(int argc, char **argv) signal(SIGINT, int_exit); sleep(duration); - profile_print_and_cleanup(); - return 0; + return profile_print_and_cleanup(); out: profile_close_perf_events(); From 8cca9a6bb68181bf50f3a9b4c54c06cd695be4dc Mon Sep 17 00:00:00 2001 From: Luis Vieira Date: Tue, 15 Sep 2026 15:13:35 -0300 Subject: [PATCH 148/660] libbpf: Fix program log buffer size validation bpf_program__set_log_buf() checks the existing prog->log_size instead of the incoming log_size argument when enforcing the UINT_MAX limit. As a result, an oversized value can be accepted. The setter also accepts a non-NULL log_buf with a zero log_size, while bpf_prog_load() rejects mismatched log buffer and size values. Validate the log buffer and size pair consistently with bpf_prog_load(), and check the supplied log_size against UINT_MAX. Signed-off-by: Luis Vieira Signed-off-by: Andrii Nakryiko Acked-by: Mykyta Yatsenko Link: https://lore.kernel.org/bpf/20260915-libbpf-log-buf-fix-v2-1-2feca3fa4842@gmail.com --- tools/lib/bpf/libbpf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 613afae26519..f7a0517d8fa8 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -9966,9 +9966,9 @@ const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_siz int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) { - if (log_size && !log_buf) + if (!!log_buf != !!log_size) return libbpf_err(-EINVAL); - if (prog->log_size > UINT_MAX) + if (log_size > UINT_MAX) return libbpf_err(-EINVAL); if (prog->obj->state >= OBJ_LOADED) return libbpf_err(-EBUSY); From 961b8946acb482b6d7a39c266d623e5f9c4e873f Mon Sep 17 00:00:00 2001 From: Mingpei CAO Date: Tue, 15 Sep 2026 16:22:36 +0000 Subject: [PATCH 149/660] libbpf: Reject local struct_ops bitfields before data access A local struct_ops type can contain a bitfield absent from the corresponding kernel BTF type. bpf_map__init_kern_struct_ops() checks whether data for an absent local member is zero. For a bitfield, member->offset contains the bit offset and field width. bpf_map__init_kern_struct_ops() used the encoded value to calculate the member data pointer before rejecting the bitfield. The zero-data check could therefore read outside st_ops->data and crash libbpf. Reject local bitfields before calculating the member data pointer. Keep the existing rejection for bitfields in the kernel type. Fixes: c911fc61a7ce ("libbpf: Skip zeroed or null fields if not found in the kernel type.") Signed-off-by: Mingpei CAO Signed-off-by: Andrii Nakryiko Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260915162252.473044-2-caomingpei@gmail.com --- tools/lib/bpf/libbpf.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index f7a0517d8fa8..33ff61b151f3 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -1223,6 +1223,12 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map) const char *mname; mname = btf__name_by_offset(btf, member->name_off); + if (btf_member_bitfield_size(type, i)) { + pr_warn("struct_ops init_kern %s: local bitfield %s is not supported\n", + map->name, mname); + return -ENOTSUP; + } + moff = member->offset / 8; mdata = data + moff; msize = btf__resolve_size(btf, member->type); @@ -1259,8 +1265,7 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map) } kern_member_idx = kern_member - btf_members(kern_type); - if (btf_member_bitfield_size(type, i) || - btf_member_bitfield_size(kern_type, kern_member_idx)) { + if (btf_member_bitfield_size(kern_type, kern_member_idx)) { pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", map->name, mname); return -ENOTSUP; From 69d38c792c53a61ee79891e488d749d4da8ab674 Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Fri, 18 Sep 2026 16:17:53 +0300 Subject: [PATCH 150/660] mm: Add copy_remote_mm_str() copy_remote_vm_str() gets the target address space from a struct task_struct. This does not work for an address space that exists but is not yet associated with a task_struct, such as the mm held by struct linux_binprm during exec. Add copy_remote_mm_str(), which operates directly on a struct mm_struct. Use a common internal interface for the MMU and NOMMU implementations and define both public wrappers in mm/util.c. Preserve the existing copy_remote_vm_str() behavior, including handling zero-length requests before acquiring the task's mm. Signed-off-by: Anastasios Papagiannis Signed-off-by: Alexei Starovoitov Acked-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Link: https://patch.msgid.link/20260918131757.42802-2-tasos.papagiannnis@gmail.com --- include/linux/mm.h | 8 +++---- mm/internal.h | 3 +++ mm/memory.c | 43 ++------------------------------- mm/nommu.c | 43 ++------------------------------- mm/util.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 86 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..63f40e615754 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3325,10 +3325,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -#ifdef CONFIG_BPF_SYSCALL -extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags); -#endif +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, diff --git a/mm/internal.h b/mm/internal.h index 38b1165212c9..7c326965a94f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -25,6 +25,9 @@ struct folio_batch; struct hstate; +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); + struct huge_bootmem_page { struct list_head list; struct hstate *hstate; diff --git a/mm/memory.c b/mm/memory.c index 8b0c2c735d3d..fa819a8a1df6 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7326,13 +7326,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len, unsigned int gup_flags) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { void *old_buf = buf; int err = 0; @@ -7408,44 +7407,6 @@ out: return buf - old_buf; } -/** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - /* * Print the name of a VMA. */ diff --git a/mm/nommu.c b/mm/nommu.c index 498e01ee40b0..68b05565e15a 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1741,13 +1741,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { unsigned long addr_end; struct vm_area_struct *vma; @@ -1782,44 +1781,6 @@ out: return ret; } -/** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour (unused) - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - /** * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode * @inode: The inode to check diff --git a/mm/util.c b/mm/util.c index bf0513d1d3d0..75c0fc2061aa 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1061,6 +1061,66 @@ out: return res; } +/** + * copy_remote_mm_str - copy a string from a remote address space. + * @mm: the remote address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * The caller must hold a reference on @mm. + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + if (unlikely(len == 0)) + return 0; + + return __copy_remote_mm_str(mm, addr, buf, len, gup_flags); +} + +/** + * copy_remote_vm_str - copy a string from another process's address space. + * @tsk: the task of the target address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + struct mm_struct *mm; + int ret; + + if (unlikely(len == 0)) + return 0; + + mm = get_task_mm(tsk); + if (!mm) { + *(char *)buf = '\0'; + return -EFAULT; + } + + ret = __copy_remote_mm_str(mm, addr, buf, len, gup_flags); + + mmput(mm); + + return ret; +} +EXPORT_SYMBOL_GPL(copy_remote_vm_str); + int __weak memcmp_pages(struct page *page1, struct page *page2) { char *addr1, *addr2; From adc7fb9955e2718faf949cdc1efaf78179978445 Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Fri, 18 Sep 2026 16:17:54 +0300 Subject: [PATCH 151/660] exec: Clear bprm->mm before dropping its reference Once mmput() drops the final reference to bprm->mm, the pointer must no longer remain accessible through struct linux_binprm. The successful exec path and the bprm initialization error path already clear bprm->mm when ownership is transferred or released. Do the same in free_bprm() before calling mmput(). This is required for BPF kfuncs where bprm->mm is either NULL or points to a live mm_struct to ensure safe access. Signed-off-by: Anastasios Papagiannis Signed-off-by: Alexei Starovoitov Reviewed-by: Sun Jian Link: https://patch.msgid.link/20260918131757.42802-3-tasos.papagiannnis@gmail.com --- fs/exec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index 745f6eb5279e..4ddd403fd91c 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1456,9 +1456,12 @@ void bprm_drop_loader(struct linux_binprm *bprm) static void free_bprm(struct linux_binprm *bprm) { - if (bprm->mm) { + struct mm_struct *mm = bprm->mm; + + if (mm) { acct_arg_size(bprm, 0); - mmput(bprm->mm); + bprm->mm = NULL; + mmput(mm); } if (bprm->user_ns) put_user_ns(bprm->user_ns); From 3f9d98dfed79ffc969d15cf214388ebf861f03af Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Fri, 18 Sep 2026 16:17:55 +0300 Subject: [PATCH 152/660] bpf: Add user memory access kfuncs for mm_struct On CONFIG_MMU kernels, when security_bprm_check() runs, the argument and environment strings for the exec have been copied into bprm->mm. The new address space is not associated with a task_struct until exec_mmap(), so existing BPF user memory helpers cannot access it. Add bpf_copy_from_user_mm() and bpf_copy_from_user_mm_str() kfuncs. Both take a struct mm_struct pointer directly, allowing callers to access trusted address spaces that are not associated with a task_struct. bpf_copy_from_user_mm() has similar semantics to bpf_copy_from_user_task(). bpf_copy_from_user_mm_str() copies one NUL-terminated string and returns its size including the NUL terminator. It accepts BPF_F_PAD_ZEROS to clear unused destination bytes on success. Refactor the task-based helpers and the new mm-based kfuncs to share static internal implementations. The task-based interfaces validate their arguments before acquiring and holding a reference to the task's mm for the copy. No behavior change is intended for the existing task-based interfaces. Register both new kfuncs and mark them KF_SLEEPABLE because accessing a remote address space can fault. Signed-off-by: Anastasios Papagiannis Signed-off-by: Alexei Starovoitov Reviewed-by: Matt Bobrowski Link: https://patch.msgid.link/20260918131757.42802-4-tasos.papagiannnis@gmail.com --- kernel/bpf/helpers.c | 154 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 142 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 051b6654e57c..82402d97ce67 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -679,9 +679,44 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +static int __bpf_copy_from_user_mm(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm) +{ + int ret; + + ret = access_remote_vm(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret == size) + return 0; + + memset(dst, 0, size); + /* Return -EFAULT for partial read */ + return ret < 0 ? ret : -EFAULT; +} + +static int __bpf_copy_from_user_mm_str(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm, u64 flags) +{ + int ret; + + ret = copy_remote_mm_str(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret < 0) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, size); + return ret; + } + + if (flags & BPF_F_PAD_ZEROS) + memset(dst + ret, 0, size - ret); + + return ret + 1; +} + BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) { + struct mm_struct *mm; int ret; /* flags is not used yet */ @@ -691,13 +726,16 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, if (unlikely(!size)) return 0; - ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0); - if (ret == size) - return 0; + mm = get_task_mm(tsk); + if (!mm) { + memset(dst, 0, size); + return -EFAULT; + } - memset(dst, 0, size); - /* Return -EFAULT for partial read */ - return ret < 0 ? ret : -EFAULT; + ret = __bpf_copy_from_user_mm(dst, size, user_ptr, mm); + mmput(mm); + + return ret; } const struct bpf_func_proto bpf_copy_from_user_task_proto = { @@ -3659,6 +3697,92 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user return ret + 1; } +/** + * bpf_copy_from_user_mm() - Copy data from an address space + * @dst: Destination address, in kernel space + * @dst__sz: Number of bytes to copy + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: Reserved for future use; must be zero + * + * Copies data from the user address space associated with @mm. The destination + * is zeroed if an attempted copy cannot be completed in full. Unsupported + * flags return -EINVAL without modifying @dst. + * + * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if @mm is no + * longer active or the copy fails or is partial. + */ +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags)) + return -EINVAL; + + if (unlikely(!dst__sz)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + memset(dst, 0, dst__sz); + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm(dst, dst__sz, unsafe_ptr__ign, mm); + mmput(mm); + + return ret; +} + +/** + * bpf_copy_from_user_mm_str() - Copy a string from an address space + * @dst: Destination address, in kernel space. This buffer must be + * at least @dst__sz bytes long + * @dst__sz: Maximum number of bytes to copy, including the trailing NUL + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: The only supported flag is BPF_F_PAD_ZEROS + * + * Copies a NUL-terminated string from the user address space associated with + * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz + * is zero. + * + * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of + * @dst is cleared on success and all of @dst is cleared on a copy failure. + * Unsupported flags return -EINVAL without modifying @dst. + * + * Return: The number of copied bytes including the NUL terminator on success, + * -EINVAL if @flags is invalid, or -EFAULT if @mm is no longer active or the + * copy fails. + */ +__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags & ~BPF_F_PAD_ZEROS)) + return -EINVAL; + + if (unlikely(dst__sz == 0)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + else + *(char *)dst = '\0'; + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); + + return ret; +} + /** * bpf_copy_from_user_task_str() - Copy a string from an task's address space * @dst: Destination address, in kernel space. This buffer must be @@ -3682,6 +3806,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, const void __user *unsafe_ptr__ign, struct task_struct *tsk, u64 flags) { + struct mm_struct *mm; int ret; if (unlikely(flags & ~BPF_F_PAD_ZEROS)) @@ -3690,17 +3815,20 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, if (unlikely(dst__sz == 0)) return 0; - ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); - if (ret < 0) { + mm = get_task_mm(tsk); + if (!mm) { if (flags & BPF_F_PAD_ZEROS) memset(dst, 0, dst__sz); - return ret; + else + *(char *)dst = '\0'; + return -EFAULT; } - if (flags & BPF_F_PAD_ZEROS) - memset(dst + ret, 0, dst__sz - ret); + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); - return ret + 1; + return ret; } /* Keep unsigned long in prototype so that kfunc is usable when emitted to @@ -4925,6 +5053,8 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) From cca46f91737ca9e41c0f86b326e5eaf695069e40 Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Fri, 18 Sep 2026 16:17:56 +0300 Subject: [PATCH 153/660] bpf: Mark linux_binprm->mm as trusted-or-null Mark linux_binprm->mm as a trusted-or-null nested pointer so BPF programs can pass it to kfuncs after a NULL check. The field is either NULL or points to a live mm_struct whenever BPF can access a linux_binprm. On successful exec, exec_mmap() installs the new address space before begin_new_exec() clears bprm->mm. The bprm_mm_init() error path clears the field before mmdrop(), and free_bprm() clears it before mmput(), as ensured by an earlier patch in this series. Update the existing LSM selftest to check bprm->mm for NULL before dereferencing it, as required for trusted-or-null pointers. Signed-off-by: Anastasios Papagiannis Signed-off-by: Alexei Starovoitov Reviewed-by: Sun Jian Reviewed-by: Matt Bobrowski Link: https://patch.msgid.link/20260918131757.42802-5-tasos.papagiannnis@gmail.com --- kernel/bpf/verifier.c | 5 +++++ tools/testing/selftests/bpf/progs/lsm.c | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6c6b8d8520cd..3539a768b921 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5997,6 +5997,10 @@ BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; +BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { + struct mm_struct *mm; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk; }; @@ -6051,6 +6055,7 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, { BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, diff --git a/tools/testing/selftests/bpf/progs/lsm.c b/tools/testing/selftests/bpf/progs/lsm.c index 7de173daf27b..7441d66c080c 100644 --- a/tools/testing/selftests/bpf/progs/lsm.c +++ b/tools/testing/selftests/bpf/progs/lsm.c @@ -113,6 +113,7 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) { __u32 pid = bpf_get_current_pid_tgid() >> 32; struct inner_map *inner_map; + struct mm_struct *mm; char args[64]; __u32 key = 0; __u64 *value; @@ -121,7 +122,9 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) bprm_count++; bpf_copy_from_user(args, sizeof(args), (void *)bprm->vma->vm_mm->arg_start); - bpf_copy_from_user(args, sizeof(args), (void *)bprm->mm->arg_start); + mm = bprm->mm; + if (mm) + bpf_copy_from_user(args, sizeof(args), (void *)mm->arg_start); value = bpf_map_lookup_elem(&array, &key); if (value) From b5501fb38ff71ead49c25d389e298293414cd39a Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Fri, 18 Sep 2026 16:17:57 +0300 Subject: [PATCH 154/660] selftests/bpf: Test mm_struct user memory kfuncs with linux_binprm Add a sleepable BPF LSM program attached to bprm_check_security to test bpf_copy_from_user_mm() and bpf_copy_from_user_mm_str(). Starting at bprm->p, verify that bpf_copy_from_user_mm() can copy the contiguous NUL-separated argument and environment data. Then use bpf_copy_from_user_mm_str() to read each argument and environment string separately, advancing the offset by the length returned from each call. Signed-off-by: Anastasios Papagiannis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260918131757.42802-6-tasos.papagiannnis@gmail.com --- .../bpf/prog_tests/copy_from_user_bprm.c | 65 +++++++++++++++++ .../selftests/bpf/progs/copy_from_user_bprm.c | 69 +++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c create mode 100644 tools/testing/selftests/bpf/progs/copy_from_user_bprm.c diff --git a/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c new file mode 100644 index 000000000000..310370689427 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include + +#include "copy_from_user_bprm.skel.h" + +void test_copy_from_user_bprm(void) +{ + char data[] = "first\0second-argument\0SOME_ENV=a\0OTHER_ENV=something"; + char arg0[] = "first"; + char arg1[] = "second-argument"; + char env0[] = "SOME_ENV=a"; + char env1[] = "OTHER_ENV=something"; + struct copy_from_user_bprm *skel; + pid_t child; + int status; + + skel = copy_from_user_bprm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (!ASSERT_OK(copy_from_user_bprm__attach(skel), "attach")) + goto out; + + child = fork(); + if (!ASSERT_GE(child, 0, "fork")) + goto out; + + if (!child) { + char *const argv[] = { arg0, arg1, NULL }; + char *const envp[] = { env0, env1, NULL }; + + skel->bss->monitored_pid = getpid(); + execvpe("true", argv, envp); + _exit(errno); + } + + if (!ASSERT_EQ(waitpid(child, &status, 0), child, "waitpid")) + goto out; + + if (ASSERT_TRUE(WIFEXITED(status), "child_exited")) + ASSERT_EQ(WEXITSTATUS(status), EPERM, "exec_errno"); + + ASSERT_EQ(skel->bss->bprm_argc, 2, "bprm_argc"); + ASSERT_EQ(skel->bss->bprm_envc, 2, "bprm_envc"); + ASSERT_EQ(skel->bss->data_len, sizeof(data), "data_len"); + ASSERT_EQ(skel->bss->invalid_flags_ret, -EINVAL, "invalid_flags_ret"); + ASSERT_EQ(skel->bss->copy_ret, 0, "copy_ret"); + ASSERT_EQ(skel->bss->arg0_ret, sizeof(arg0), "arg0_ret"); + ASSERT_EQ(skel->bss->arg1_ret, sizeof(arg1), "arg1_ret"); + ASSERT_EQ(skel->bss->env0_ret, sizeof(env0), "env0_ret"); + ASSERT_EQ(skel->bss->env1_ret, sizeof(env1), "env1_ret"); + ASSERT_EQ(memcmp(skel->bss->data, data, sizeof(data)), 0, "data"); + ASSERT_EQ(memcmp(skel->bss->arg0, arg0, sizeof(arg0)), 0, "arg0"); + ASSERT_EQ(memcmp(skel->bss->arg1, arg1, sizeof(arg1)), 0, "arg1"); + ASSERT_EQ(memcmp(skel->bss->env0, env0, sizeof(env0)), 0, "env0"); + ASSERT_EQ(memcmp(skel->bss->env1, env1, sizeof(env1)), 0, "env1"); + +out: + copy_from_user_bprm__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c new file mode 100644 index 000000000000..00ecba3ea567 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" + +#include +#include +#include + +char _license[] SEC("license") = "GPL"; + +int monitored_pid; +int bprm_argc; +int bprm_envc; +int data_len; +int invalid_flags_ret; +int copy_ret; +int arg0_ret; +int arg1_ret; +int env0_ret; +int env1_ret; +char data[64] = {}; +char arg0[32] = {}; +char arg1[32] = {}; +char env0[32] = {}; +char env1[32] = {}; + +SEC("lsm.s/bprm_check_security") +int BPF_PROG(check_exec_args, struct linux_binprm *bprm) +{ + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct mm_struct *mm; + u64 offset = 0; + + if (pid != monitored_pid) + return 0; + + mm = bprm->mm; + if (!mm) + return 0; + + bprm_argc = bprm->argc; + bprm_envc = bprm->envc; + + /* this is the total size of args and envs starting from bprm->p */ + data_len = bprm->exec - bprm->p; + + invalid_flags_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, ~0ULL); + + copy_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, 0); + if (copy_ret) + return 0; + + /* arg0 is at bprm->p */ + arg0_ret = bpf_copy_from_user_mm_str(arg0, sizeof(arg0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg0_ret; + + /* arg1 is at bprm->p + arg0_ret */ + arg1_ret = bpf_copy_from_user_mm_str(arg1, sizeof(arg1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg1_ret; + + /* env0 is at bprm->p + arg0_ret + arg1_ret */ + env0_ret = bpf_copy_from_user_mm_str(env0, sizeof(env0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += env0_ret; + + /* env1 is at bprm->p + arg0_ret + arg1_ret + env0_ret */ + env1_ret = bpf_copy_from_user_mm_str(env1, sizeof(env1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + + return -EPERM; +} From 01208951214460946b3c71d442663c65a8f5a96f Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:27 -0700 Subject: [PATCH 155/660] bpf: Remove __rcu tagging in st_link->map st_link->map is always written under update_mutex. The paths that read st_link->map with rcu_read_lock() are not in the fast path, so they can simply take update_mutex instead. Remove the __rcu annotation and replace all RCU accessors with direct pointer reads under update_mutex. Use READ_ONCE() in bpf_struct_ops_map_link_poll() which reads the pointer without holding update_mutex. It is a simplification change. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Reviewed-by: Eduard Zingerman Link: https://patch.msgid.link/20260917200542.3689605-2-ameryhung@gmail.com --- kernel/bpf/bpf_struct_ops.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index d7c3030bc63b..3935bf35a423 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -57,7 +57,7 @@ struct bpf_struct_ops_map { struct bpf_struct_ops_link { struct bpf_link link; - struct bpf_map __rcu *map; + struct bpf_map *map; wait_queue_head_t wait_hup; }; @@ -1297,8 +1297,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) struct bpf_struct_ops_map *st_map; st_link = container_of(link, struct bpf_struct_ops_link, link); - st_map = (struct bpf_struct_ops_map *) - rcu_dereference_protected(st_link->map, true); + st_map = (struct bpf_struct_ops_map *)st_link->map; if (st_map) { st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); bpf_map_put(&st_map->map); @@ -1313,11 +1312,11 @@ static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) seq_printf(seq, "map_id:\t%d\n", map->id); - rcu_read_unlock(); + mutex_unlock(&update_mutex); } static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, @@ -1327,11 +1326,11 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) info->struct_ops.map_id = map->id; - rcu_read_unlock(); + mutex_unlock(&update_mutex); return 0; } @@ -1354,7 +1353,7 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map mutex_lock(&update_mutex); - old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); + old_map = st_link->map; if (!old_map) { err = -ENOLINK; goto err_out; @@ -1376,7 +1375,7 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map goto err_out; bpf_map_inc(new_map); - rcu_assign_pointer(st_link->map, new_map); + WRITE_ONCE(st_link->map, new_map); bpf_map_put(old_map); err_out: @@ -1393,7 +1392,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) mutex_lock(&update_mutex); - map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); + map = st_link->map; if (!map) { mutex_unlock(&update_mutex); return 0; @@ -1402,7 +1401,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); - RCU_INIT_POINTER(st_link->map, NULL); + WRITE_ONCE(st_link->map, NULL); /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or * bpf_map_inc() in bpf_struct_ops_map_link_update(). */ @@ -1422,7 +1421,7 @@ static __poll_t bpf_struct_ops_map_link_poll(struct file *file, poll_wait(file, &st_link->wait_hup, pts); - return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP; + return READ_ONCE(st_link->map) ? 0 : EPOLLHUP; } static const struct bpf_link_ops bpf_struct_ops_map_lops = { @@ -1478,7 +1477,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) link = NULL; goto err_out; } - RCU_INIT_POINTER(link->map, map); + link->map = map; mutex_unlock(&update_mutex); return bpf_link_settle(&link_primer); From 5db69b0fbdc88150e80c1b904f5201502c365b35 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:28 -0700 Subject: [PATCH 156/660] bpf: Make struct_ops tasks_rcu grace period optional bpf_struct_ops_map_free() currently waits for both a regular RCU grace period and a tasks RCU grace period for every struct_ops map through synchronize_rcu_mult(call_rcu, call_rcu_tasks). A regular RCU grace period is still required for all struct_ops maps because the struct_ops trampoline ksyms requires a rcu grace period (take a look at the list_del_rcu in __bpf_ksym_del). Add a map_free_pre_rcu() callback so the struct_ops map can remove ksyms before bpf_map_put() wait for the regular rcu grace period. The tasks RCU grace period is only needed by tcp_congestion_ops. Add free_after_tasks_rcu_gp only to struct bpf_struct_ops instead of the bpf_map. When CONFIG_TASKS_RCU=n, synchronize_rcu_tasks() is the same as synchronize_rcu(). Since all struct_ops maps now complete a regular RCU grace period before bpf_struct_ops_map_free() runs, skip the extra synchronize_rcu_tasks() call in this case. This cleanup prepares for a later patch that needs to support free_after_mult_rcu_gp. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Reviewed-by: Eduard Zingerman Link: https://patch.msgid.link/20260917200542.3689605-3-ameryhung@gmail.com --- include/linux/bpf.h | 7 +++++++ kernel/bpf/bpf_struct_ops.c | 31 +++++++++++++------------------ kernel/bpf/syscall.c | 3 +++ net/ipv4/bpf_tcp_ca.c | 16 ++++++++++++++++ 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2a5fa346aada..1198404885c8 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -90,6 +90,7 @@ struct bpf_map_ops { struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); + void (*map_free_pre_rcu)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); @@ -2131,6 +2132,11 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for + * a tasks_rcu gp before freeing the struct_ops map + * and its progs. It is unnecessary if the @unreg + * has waited for the correct rcu gp or the @unreg + * has ensured all struct_ops prog has finished running. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2149,6 +2155,7 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + bool free_after_tasks_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 3935bf35a423..7802859eac1e 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1024,9 +1024,18 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_map_area_free(st_map); } +static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + bpf_struct_ops_map_del_ksyms(st_map); +} + static void bpf_struct_ops_map_free(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + struct bpf_struct_ops *st_ops = st_map->st_ops_desc->st_ops; + bool tasks_rcu = st_ops->free_after_tasks_rcu_gp; /* st_ops->owner was acquired during map_alloc to implicitly holds * the btf's refcnt. The acquire was only done when btf_is_module() @@ -1037,24 +1046,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_dissoc_progs(st_map); - bpf_struct_ops_map_del_ksyms(st_map); - - /* The struct_ops's function may switch to another struct_ops. - * - * For example, bpf_tcp_cc_x->init() may switch to - * another tcp_cc_y by calling - * setsockopt(TCP_CONGESTION, "tcp_cc_y"). - * During the switch, bpf_struct_ops_put(tcp_cc_x) is called - * and its refcount may reach 0 which then free its - * trampoline image while tcp_cc_x is still running. - * - * A vanilla rcu gp is to wait for all bpf-tcp-cc prog - * to finish. bpf-tcp-cc prog is non sleepable. - * A rcu_tasks gp is to wait for the last few insn - * in the tramopline image to finish before releasing - * the trampoline image. - */ - synchronize_rcu_mult(call_rcu, call_rcu_tasks); + if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) + synchronize_rcu_tasks(); __bpf_struct_ops_map_free(map); } @@ -1163,6 +1156,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); + map->free_after_rcu_gp = true; return map; @@ -1195,6 +1189,7 @@ const struct bpf_map_ops bpf_struct_ops_map_ops = { .map_alloc_check = bpf_struct_ops_map_alloc_check, .map_alloc = bpf_struct_ops_map_alloc, .map_free = bpf_struct_ops_map_free, + .map_free_pre_rcu = bpf_struct_ops_map_free_pre_rcu, .map_get_next_key = bpf_struct_ops_map_get_next_key, .map_lookup_elem = bpf_struct_ops_map_lookup_elem, .map_delete_elem = bpf_struct_ops_map_delete_elem, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index def57bddb092..01a1f1dd3b67 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -954,6 +954,9 @@ void bpf_map_put(struct bpf_map *map) /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); + if (map->ops->map_free_pre_rcu) + map->ops->map_free_pre_rcu(map); + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); /* RCU tasks trace grace period implies RCU grace period. */ if (READ_ONCE(map->free_after_mult_rcu_gp)) diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 791e15063237..e224ecafbd69 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -339,6 +339,22 @@ static struct bpf_struct_ops bpf_tcp_congestion_ops = { .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", .cfi_stubs = &__bpf_ops_tcp_congestion_ops, + /* The struct_ops's function may switch to another struct_ops. + * + * For example, bpf_tcp_cc_x->init() may switch to + * another tcp_cc_y by calling + * setsockopt(TCP_CONGESTION, "tcp_cc_y"). + * During the switch, bpf_struct_ops_put(tcp_cc_x) is called + * and its refcount may reach 0 which then free its + * trampoline image while tcp_cc_x is still running. + * + * A vanilla rcu gp is to wait for all bpf-tcp-cc prog + * to finish. bpf-tcp-cc prog is non sleepable. + * A rcu_tasks gp is to wait for the last few insn + * in the tramopline image to finish before releasing + * the trampoline image. + */ + .free_after_tasks_rcu_gp = true, .owner = THIS_MODULE, }; From 3668bc410260fbd97784c3c4cdc73cf06f86f940 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:29 -0700 Subject: [PATCH 157/660] bpf: Add bpf_struct_ops accessor helpers Add the helper functions bpf_struct_ops_map_kdata() and bpf_struct_ops_map_cfi_stubs() in bpf_struct_ops.c. They will be called from cgroup.c in the upcoming patch to create a struct_ops to cgroup attachment link. bpf_struct_ops_valid_to_reg() is also exposed for the upcoming caller in cgroup.c. The link update validation is also refactored into a new function bpf_struct_ops_link_update_check() such that it can be reused by the caller in cgroup.c in the upcoming patch. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-4-ameryhung@gmail.com --- include/linux/bpf.h | 27 +++++++++++++++++++ kernel/bpf/bpf_struct_ops.c | 53 +++++++++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1198404885c8..90ab467d25a2 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2290,6 +2290,11 @@ u32 bpf_struct_ops_id(const void *kdata); int bpf_struct_ops_for_each_prog(const void *kdata, int (*cb)(struct bpf_prog *prog, void *data), void *data); +void *bpf_struct_ops_map_kdata(struct bpf_map *map); +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, + struct bpf_map *expected_old_map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2354,6 +2359,28 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } +static inline u32 bpf_struct_ops_id(void *kdata) +{ + return 0; +} +static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + return NULL; +} +static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + return NULL; +} +static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +{ + return false; +} +static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + return -EOPNOTSUPP; +} #endif diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 7802859eac1e..12ce9654ffdd 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1276,7 +1276,23 @@ int bpf_struct_ops_for_each_prog(const void *kdata, } EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog); -static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->kvalue.data; +} + +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cfi_stubs; +} + +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; @@ -1329,6 +1345,26 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, return 0; } +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + struct bpf_struct_ops_map *st_map, *old_st_map; + + if (!old_map) + return -ENOLINK; + if (expected_old_map && old_map != expected_old_map) + return -EPERM; + + st_map = container_of(new_map, struct bpf_struct_ops_map, map); + old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); + /* The new and old struct_ops must be the same type. */ + if (st_map->st_ops_desc != old_st_map->st_ops_desc) + return -EINVAL; + + return 0; +} + static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map, struct bpf_map *expected_old_map) { @@ -1347,23 +1383,12 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map return -EOPNOTSUPP; mutex_lock(&update_mutex); - old_map = st_link->map; - if (!old_map) { - err = -ENOLINK; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) goto err_out; - } - if (expected_old_map && old_map != expected_old_map) { - err = -EPERM; - goto err_out; - } old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); - /* The new and old struct_ops must be the same type. */ - if (st_map->st_ops_desc != old_st_map->st_ops_desc) { - err = -EINVAL; - goto err_out; - } err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link); if (err) From 1ef6cb2cc587fae652e98e695f60434932337223 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:30 -0700 Subject: [PATCH 158/660] bpf: Remove unnecessary prog_list_prog() check effective_prog_pos(), called from replace_effective_prog() and purge_effective_progs(), tests "!prog_list_prog(pl)" to skip a 'detaching' pl. When detaching a pl, pl->prog and pl->link are set to NULL in case the update_effective_progs() failed. However, replace_effective_prog() is not detaching a pl, so the case "!prog_list_prog()" will not happen. In purge_effective_prog(), the pl->prog and pl->link are restored before calling purge_effective_progs(), so the case "!prog_list_prog()" will not happen either. This patch removes them as a prep work for the upcoming work in attaching struct_ops to cgroup. When attaching a struct_ops to cgroup, there is a link->map case and the prog_list_prog() will not consider the link->map. The replace_effective_prog() and purge_effective_progs() will then incorrectly skip a pl with struct_ops map attached to it. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-5-ameryhung@gmail.com --- kernel/bpf/cgroup.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 149672c76c49..2fee0cbe5dfd 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -973,9 +973,10 @@ static int effective_prog_pos(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) - continue; - + /* + * No detaching pl (NULL prog and link) is visible to the callers, + * so skip the check compute_effective_progs() needs. + */ if (pl->flags & BPF_F_PREORDER) { if (pl == target_pl) pos = bstart; From c6dde091dc12f71d479d25345872b9aa3020fc60 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:31 -0700 Subject: [PATCH 159/660] bpf: Replace prog_list_prog() check with direct pl->prog and pl->link check prog_list_length() and compute_effective_progs() use !prog_list_prog(pl) to skip a 'detaching' pl. When pl->link is not NULL, prog_list_prog(pl) returns the pl->link->link.prog. This does not work for the upcoming struct_ops patch where pl->link is not NULL but pl->link->link.prog is NULL, because a struct_ops map is attached to the cgroup instead of a BPF prog. To prepare for the upcoming struct_ops patch, this patch replaces the prog_list_prog() test with the "!pl->prog && !pl->link". In __cgroup_bpf_detach(), both pl->prog and pl->link are set to NULL, so testing "!pl->prog && !pl->link" is the same test to tell if a pl is being detached. This change should be a no-op. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260917200542.3689605-6-ameryhung@gmail.com --- kernel/bpf/cgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 2fee0cbe5dfd..6d865e765d58 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -408,7 +408,7 @@ static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt) u32 cnt = 0; hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (preorder_cnt && (pl->flags & BPF_F_PREORDER)) (*preorder_cnt)++; @@ -482,7 +482,7 @@ static int compute_effective_progs(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (pl->flags & BPF_F_PREORDER) { From 4d71b3e0821f71819090f1ab422937e543b839a8 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:32 -0700 Subject: [PATCH 160/660] bpf: Add prog_list_init_item(), prog_list_replace_item(), and prog_list_id() Add three helpers to abstract operations on a bpf_prog_list entry. Right now, bpf_prog_array_item is initialized from prog_list_prog(pl), which returns either pl->prog or pl->link->link.prog. This will not work when struct_ops is attached to a cgroup because the attachment is backed by a struct_ops map instead of a BPF prog. The same applies to __cgroup_bpf_query(). Instead of always copying a prog id to userspace, struct_ops cgroup attachment will need to copy the struct_ops map id. Refactor bpf_prog_array_item initialization into prog_list_init_item() and prog_list_replace_item(), and refactor id lookup into prog_list_id(). These helpers will be extended to support pl->link->map in a later patch. This is a no-op change. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-7-ameryhung@gmail.com --- kernel/bpf/cgroup.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 6d865e765d58..bced8e20fa6d 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -399,6 +399,22 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) return NULL; } +static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + item->prog = prog_list_prog(pl); + bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); +} + +static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + WRITE_ONCE(item->prog, pl->link->link.prog); +} + +static u32 prog_list_id(struct bpf_prog_list *pl) +{ + return prog_list_prog(pl)->aux->id; +} + /* count number of elements in the list. * it's slow but the list cannot be long */ @@ -492,9 +508,7 @@ static int compute_effective_progs(struct cgroup *cgrp, item = &progs->items[fstart]; fstart++; } - item->prog = prog_list_prog(pl); - bpf_cgroup_storages_assign(item->cgroup_storage, - pl->storage); + prog_list_init_item(pl, item); cnt++; } @@ -1023,7 +1037,7 @@ static void replace_effective_prog(struct cgroup *cgrp, desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, pl->link->link.prog); + prog_list_replace_item(pl, item); } } @@ -1343,15 +1357,13 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } else { struct hlist_head *progs; struct bpf_prog_list *pl; - struct bpf_prog *prog; u32 id; progs = &cgrp->bpf.progs[atype]; cnt = min_t(int, prog_list_length(progs, NULL), total_cnt); i = 0; hlist_for_each_entry(pl, progs, node) { - prog = prog_list_prog(pl); - id = prog->aux->id; + id = prog_list_id(pl); if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; if (++i == cnt) From 2ef54525d2fc24224c0317e2e662307fcbbd1f9c Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:33 -0700 Subject: [PATCH 161/660] bpf: Move LSM trampoline unlink into bpf_cgroup_link_auto_detach() Move the LSM trampoline unlink into bpf_cgroup_link_auto_detach(). The purpose is to consolidate the auto_detach cleanup logic. It prepares for the upcoming struct_ops cgroup attachment patch where bpf_cgroup_link_auto_detach() will need to handle the struct_ops case (link->map != NULL). This is a no-op change. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-8-ameryhung@gmail.com --- kernel/bpf/cgroup.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index bced8e20fa6d..0ce58764caae 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -313,6 +313,8 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], */ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) { + if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(link->link.prog); cgroup_put(link->cgroup); link->cgroup = NULL; } @@ -346,11 +348,8 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_trampoline_unlink_cgroup_shim(pl->prog); bpf_prog_put(pl->prog); } - if (pl->link) { - if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP) - bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog); + if (pl->link) bpf_cgroup_link_auto_detach(pl->link); - } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); } From e176821b68d8022180ac7ecce25dff797e66ca97 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:34 -0700 Subject: [PATCH 162/660] bpf: Add a few bpf_cgroup_array_* helper functions In the upcoming patch, the array can store a struct_ops map. The array could have a cfi_stubs acting as a dummy instead of the dummy_bpf_prog. The array logic will need to skip the cfi_stubs also in order to support storing struct_ops map in the array. bpf_cgroup_array_length(), bpf_cgroup_array_copy_to_user(), and bpf_cgroup_array_delete_safe_at() are added as a preparation work to allow skipping the cfi_stubs in the upcoming patch. This patch only skips the dummy_bpf_prog which is the same as the existing behavior. The current bpf_prog_array_*() callers are changed to call the new bpf_cgroup_array_*(). This is a no-op change. Unlike bpf_prog_array_copy_to_user(), bpf_cgroup_array_copy_to_user() does not need a temporary buffer. The cgroup caller already holds cgroup_mutex and dereferences the effective array with rcu_dereference_protected(), so it does not copy to userspace from an RCU read-side critical section. Details in commit 0911287ce32b. Another addition is the bpf_cgroup_array_free(). This prepares the array to have a different rcu gp for the struct_ops use case, for example, a struct_ops could have mix of sleepable ops and non-sleepable ops. In this patch, bpf_cgroup_array_free() only goes through the regular rcu gp. This is a no-op change also. bpf_prog_dummy() is also added to return the global dummy_bpf_prog. bpf_cgroup_array_dummy() is added to decide the sentinel based on atype. It now always returns bpf_prog_dummy(). In the upcoming patch, it can return a cfi_stubs if the atype belongs to a struct_ops. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-9-ameryhung@gmail.com --- include/linux/bpf.h | 1 + kernel/bpf/cgroup.c | 80 ++++++++++++++++++++++++++++++++++++++++----- kernel/bpf/core.c | 5 +++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 90ab467d25a2..b56e2a5ca548 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2598,6 +2598,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); +struct bpf_prog *bpf_prog_dummy(void); struct bpf_run_ctx {}; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 0ce58764caae..0c8f6f6be049 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -319,6 +319,68 @@ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) link->cgroup = NULL; } +static void bpf_cgroup_array_free(struct bpf_prog_array *array) +{ + if (!array || array == &bpf_empty_prog_array) + return; + kfree_rcu(array, rcu); +} + +static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) +{ + return bpf_prog_dummy(); +} + +static int bpf_cgroup_array_length(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int cnt = 0; + + for (item = array->items; item->prog; item++) { + if (item->prog != bpf_cgroup_array_dummy(atype)) + cnt++; + } + + return cnt; +} + +static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, + __u32 __user *prog_ids, int cnt, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int i = 0; + u32 id; + + for (item = array->items; item->prog && i < cnt; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + id = item->prog->aux->id; + if (copy_to_user(prog_ids + i, &id, sizeof(id))) + return -EFAULT; + i++; + } + return item->prog ? -ENOSPC : 0; +} + +static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, + int index, enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + + for (item = array->items; item->prog; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + if (!index) { + WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype)); + return 0; + } + index--; + } + return -ENOENT; +} + /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data @@ -356,7 +418,7 @@ static void cgroup_bpf_release(struct work_struct *work) old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -530,7 +592,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array); } /** @@ -570,7 +632,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_prog_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i]); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -625,7 +687,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } continue; @@ -644,7 +706,7 @@ cleanup: css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } @@ -1191,7 +1253,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, lockdep_is_held(&cgroup_mutex)); /* Remove the program from the array */ - WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos), + WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), "Failed to purge a prog from array at index %d", pos); } } @@ -1321,7 +1383,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - total_cnt += bpf_prog_array_length(effective); + total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } @@ -1351,8 +1413,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); - ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt); + cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); + ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); } else { struct hlist_head *progs; struct bpf_prog_list *pl; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 4e208cc94752..227211166dcc 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2784,6 +2784,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); } +struct bpf_prog *bpf_prog_dummy(void) +{ + return &dummy_bpf_prog.prog; +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; From 369d9dcd8fb855f103a83454f0e1cd1cbc9118be Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:35 -0700 Subject: [PATCH 163/660] bpf: Add infrastructure to support attaching struct_ops to cgroups This patch adds necessary infrastructure to attach a struct_ops map to a cgroup. The initial need was to support migrating the legacy BPF_PROG_TYPE_SOCK_OPS to a struct_ops. Recently, there are other struct_ops use cases that need to attach struct_ops to a cgroup. For example, the recent BPF OOM and memcg discussion in LSFMMBPF 2026. The motivation is to create a consistent expectation for attaching struct_ops to cgroup instead of each subsystem creating its own infrastructure. This logic includes hierarchy expectation, ordering expectation, attachment API, and rcu gp. There is already an existing implementation for attaching multiple bpf progs to a cgroup. There are also tools built around it for querying. Attaching a struct_ops map (which is a group of bpf programs) could also adhere to a similar API and potentially reuse most of the existing implementation. A couple of ideas have been tried. One of them is to use mprog.c. In terms of the amount of changes, I eventually came to the same conclusion as in commit 120933984460 ("bpf: Implement mprog API on top of existing cgroup progs"). I then shifted the focus to reusing the current {update,compute,activate,purge}_effective_progs() which has the main logic that implements the mprog API. Since then, I tried to add a 'struct cgroup *cgroup' member to the existing 'struct bpf_struct_ops_link' and link_create will create a 'struct bpf_struct_ops_link' object to be stored in the pl->link. This turns out to have more changes on both cgroup.c and bpf_struct_ops.c than I like. This patch directly reuses the 'struct bpf_cgroup_link' which cgroup.c already understands. Add 'struct bpf_map *map' to 'struct bpf_cgroup_link'. In the future, as more subsystems are extended by struct_ops, we may consider to make 'struct bpf_map *map' as a primary citizen of a link like 'struct bpf_prog *prog' and directly add 'struct bpf_map *map' to the generic 'struct bpf_link'. The pl->link could be the traditional 'prog' link or the new 'map' link. The places that need to handle them differently have already been refactored into the new prog_list_*() added in the earlier patch. In those new prog_list_*(), this patch will check "pl->link && pl->link->map", learn that it is a 'map' link and handle it correctly. The bpf_prog_array also needs to handle that its item can store the traditional 'prog' or it can store a struct_ops map. The places that need to handle them differently have also been refactored into the new bpf_cgroup_array_*() added in the earlier patch. The two differences are: - different sentinel (dummy_bpf_prog in prog vs cfi_stub in struct_ops) - the array for struct_ops may need to go through different rcu gp. The bpf_cgroup_array_*() functions use the cgroup_bpf_attach_type (ie atype) to distinguish the array is storing prog or storing struct_ops map. This patch also implements a separate struct bpf_link_ops "cgroup_struct_ops_link_ops" to have a separate link_ops implementation that only handles the cgroup's struct_ops link. Questions: - Although this patch did not change it, it is not obvious to me how the replace_effective_progs() and purge_effective_progs() handle cases when there are existing BPF_F_PREORDER progs attached in the hlist. Misc notes: - CGROUP_TCP_SOCK_OPS is added to the 'enum cgroup_bpf_attach_type'. The actual implementation of the tcp_bpf_ops (a struct_ops) will be added in the next patch. - free_after_mult_rcu_gp is added to 'struct bpf_struct_ops' such that the bpf_prog_array can have a mix of sleepable and non-sleepable prog in a struct_ops. This can tell how the bpf_prog_array should be freed. - For a struct_ops that supports cgroup attachment, it does not need to implement its own reg/unreg function. reg/unreg to a cgroup is done by the common infrastructure added in this patch. - The cgroup's struct_ops link only supports BPF_F_ALLOW_MULTI. This is enforced internally in cgroup_bpf_struct_ops_attach. This should be consistent with the current prog's link behavior in cgroup_bpf_link_attach. In the future, we may allow each subsystem to choose differently. - A cgroup_atype member is added to 'struct bpf_struct_ops'. When a subsystem struct_ops needs to support cgroup attachment, it needs to add a value to 'enum cgroup_bpf_attach_type' and then assign it to the newly added cgroup_atype member in the bpf_struct_ops. - During LINK_CREATE in syscall, the patch uses the same BPF_STRUCT_OPS (in attr->link_create.attach_type). The bpf_struct_ops_link_create learns the map and from the map it learns the st_ops. If the st_ops->cgroup_atype is not 0, it will create a cgroup's link. - When a subsystem registers a struct_ops that supports cgroup attachment, the struct_ops infrastructure will also ask the cgroup infrastructure to remember a few things. This is done by calling cgroup_bpf_struct_ops_register(). Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260917200542.3689605-10-ameryhung@gmail.com --- include/linux/bpf-cgroup-defs.h | 1 + include/linux/bpf-cgroup.h | 28 +++ include/linux/bpf.h | 19 +- include/uapi/linux/bpf.h | 4 +- kernel/bpf/bpf_struct_ops.c | 29 +++ kernel/bpf/btf.c | 31 ++- kernel/bpf/cgroup.c | 378 +++++++++++++++++++++++++++++++- kernel/bpf/syscall.c | 1 + tools/include/uapi/linux/bpf.h | 4 +- 9 files changed, 480 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index c9e6b26abab6..0147b8bec973 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type { CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_TCP_SOCK_OPS, CGROUP_LSM_START, CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 4d0cc65976a1..8a75a6cd7309 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -100,6 +100,8 @@ struct bpf_cgroup_storage { struct bpf_cgroup_link { struct bpf_link link; struct cgroup *cgroup; + struct bpf_map *map; + wait_queue_head_t wait_hup; }; struct bpf_prog_list { @@ -110,6 +112,18 @@ struct bpf_prog_list { u32 flags; }; +#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype) \ + for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\ + ((var) = READ_ONCE(item->kdata)); \ + item++) + +static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype) +{ + return atype == CGROUP_TCP_SOCK_OPS; +} +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace); +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr); + void __init cgroup_bpf_lifetime_notifier_init(void); int __cgroup_bpf_run_filter_skb(struct sock *sk, @@ -479,6 +493,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } +static inline bool cgroup_bpf_is_struct_ops_atype(int atype) +{ + return false; +} +static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, + bool mult_trace) +{ +} +static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, + const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + #define cgroup_bpf_enabled(atype) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; }) #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; }) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b56e2a5ca548..f34b410f903e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2132,11 +2132,18 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment. + * 0 means the struct_ops type does not support cgroup attachment. + * If cgroup_atype is non-zero, the @reg and @unreg must be NULL + * because the attachment/detachment will be handled by the bpf core. * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for * a tasks_rcu gp before freeing the struct_ops map * and its progs. It is unnecessary if the @unreg * has waited for the correct rcu gp or the @unreg * has ensured all struct_ops prog has finished running. + * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for + * both tasks_trace_rcu and regular rcu grace period. + * It is usually needed if the struct_ops has sleepable prog. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2155,7 +2162,9 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + int cgroup_atype; bool free_after_tasks_rcu_gp; + bool free_after_mult_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not @@ -2295,6 +2304,7 @@ void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, struct bpf_map *expected_old_map); +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2367,6 +2377,10 @@ static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) { return NULL; } +static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + return 0; +} static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) { return NULL; @@ -2556,7 +2570,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { - struct bpf_prog *prog; + union { + struct bpf_prog *prog; + void *kdata; + }; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 732b35cc08d1..dabe01cd3def 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1756,7 +1756,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -6890,6 +6890,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 12ce9654ffdd..1178acd72296 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -13,6 +13,7 @@ #include #include #include +#include struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; @@ -1116,6 +1117,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) goto errout; } + if (st_ops_desc->st_ops->cgroup_atype && !(attr->map_flags & BPF_F_LINK)) { + ret = -EOPNOTSUPP; + goto errout; + } + vt = st_ops_desc->value_type; if (attr->value_size != vt->size) { ret = -EINVAL; @@ -1156,6 +1162,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); + map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; map->free_after_rcu_gp = true; return map; @@ -1284,6 +1291,14 @@ void *bpf_struct_ops_map_kdata(struct bpf_map *map) return st_map->kvalue.data; } +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cgroup_atype; +} + void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) { struct bpf_struct_ops_map *st_map; @@ -1459,6 +1474,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) struct bpf_link_primer link_primer; struct bpf_struct_ops_map *st_map; struct bpf_map *map; + int cgroup_atype; int err; map = bpf_map_get(attr->link_create.map_fd); @@ -1472,6 +1488,19 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) goto err_out; } + cgroup_atype = st_map->st_ops_desc->st_ops->cgroup_atype; + if (cgroup_atype) { + err = cgroup_bpf_struct_ops_attach(map, attr); + bpf_map_put(map); + return err; + } + + if (memchr_inv(&attr->link_create.cgroup, 0, sizeof(attr->link_create.cgroup)) || + attr->link_create.target_fd) { + err = -EINVAL; + goto err_out; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7daf4c286c9b..e3fb45dcee39 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -9992,6 +9993,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, struct bpf_verifier_log *log) { struct btf_struct_ops_tab *tab, *new_tab; + int cgroup_atype; int i, err; tab = btf->struct_ops_tab; @@ -10003,8 +10005,10 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, btf->struct_ops_tab = tab; } + cgroup_atype = st_ops->cgroup_atype; for (i = 0; i < tab->cnt; i++) - if (tab->ops[i].st_ops == st_ops) + if (tab->ops[i].st_ops == st_ops || + (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype)) return -EEXIST; if (tab->cnt == tab->capacity) { @@ -10024,6 +10028,31 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, if (err) return err; + if (cgroup_atype) { + /* + * Cgroup struct_ops callers hold the RCU read lock around the + * entire trampoline call, including its trailing instructions. + * A regular RCU grace period therefore protects both the kdata + * and the trampoline image, so a tasks RCU grace period is not + * needed. + */ + if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) || + st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) { + bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]); + return -EINVAL; + } + + /* + * There is no need to unregister from the cgroup when btf_free() + * runs. No struct_ops map or cgroup link can be created once its + * BTF is gone. + */ + cgroup_bpf_struct_ops_register(cgroup_atype, + tab->ops[btf->struct_ops_tab->cnt].type_id, + st_ops->cfi_stubs, + st_ops->free_after_mult_rcu_gp); + } + btf->struct_ops_tab->cnt++; return 0; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 0c8f6f6be049..696b27383974 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -24,6 +24,33 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE); EXPORT_SYMBOL(cgroup_bpf_enabled_key); +struct cgroup_struct_ops { + u32 type_id; + void *cfi_stubs; + bool mult_rcu; +}; + +static struct cgroup_struct_ops cgroup_struct_ops[MAX_CGROUP_BPF_ATTACH_TYPE]; + +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_rcu) +{ + cgroup_struct_ops[atype].type_id = type_id; + cgroup_struct_ops[atype].cfi_stubs = cfi_stubs; + cgroup_struct_ops[atype].mult_rcu = mult_rcu; +} + +static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id) +{ + enum cgroup_bpf_attach_type atype; + + for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) { + if (cgroup_bpf_is_struct_ops_atype(atype) && + cgroup_struct_ops[atype].type_id == type_id) + return atype; + } + return CGROUP_BPF_ATTACH_TYPE_INVALID; +} + /* * cgroup bpf destruction makes heavy use of work items and there can be a lot * of concurrent destructions. Use a separate workqueue so that cgroup bpf @@ -306,6 +333,19 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], bpf_cgroup_storage_link(storages[stype], cgrp, attach_type); } +static void cgroup_struct_ops_link_detach_wake(struct bpf_cgroup_link *link, bool wake_poll) +{ + cgroup_put(link->cgroup); + link->cgroup = NULL; + + bpf_map_put(link->map); + /* READ_ONCE in cgroup_struct_ops_link_poll */ + WRITE_ONCE(link->map, NULL); + + if (wake_poll) + wake_up_interruptible_poll(&link->wait_hup, EPOLLHUP); +} + /* Called when bpf_cgroup_link is auto-detached from dying cgroup. * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It * doesn't free link memory, which will eventually be done by bpf_link's @@ -313,21 +353,38 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], */ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) { + if (link->map) { + cgroup_struct_ops_link_detach_wake(link, true); + return; + } + if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP) bpf_trampoline_unlink_cgroup_shim(link->link.prog); cgroup_put(link->cgroup); link->cgroup = NULL; } -static void bpf_cgroup_array_free(struct bpf_prog_array *array) +static void bpf_cgroup_array_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct bpf_prog_array, rcu)); +} + +static void bpf_cgroup_array_free(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) { if (!array || array == &bpf_empty_prog_array) return; - kfree_rcu(array, rcu); + if (cgroup_struct_ops[atype].mult_rcu) + /* RCU tasks trace grace period implies RCU grace period. */ + call_rcu_tasks_trace(&array->rcu, bpf_cgroup_array_free_rcu); + else + kfree_rcu(array, rcu); } static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) { + if (cgroup_bpf_is_struct_ops_atype(atype)) + return cgroup_struct_ops[atype].cfi_stubs; return bpf_prog_dummy(); } @@ -356,7 +413,12 @@ static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, for (item = array->items; item->prog && i < cnt; item++) { if (item->prog == bpf_cgroup_array_dummy(atype)) continue; - id = item->prog->aux->id; + + if (cgroup_bpf_is_struct_ops_atype(atype)) + id = bpf_struct_ops_id(item->kdata); + else + id = item->prog->aux->id; + if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; i++; @@ -418,7 +480,7 @@ static void cgroup_bpf_release(struct work_struct *work) old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_cgroup_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -460,19 +522,34 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) return NULL; } +static bool prog_list_is_struct_ops(const struct bpf_prog_list *pl) +{ + return pl->link && pl->link->map; +} + static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) { + if (prog_list_is_struct_ops(pl)) { + item->kdata = bpf_struct_ops_map_kdata(pl->link->map); + return; + } + item->prog = prog_list_prog(pl); bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); } static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) { - WRITE_ONCE(item->prog, pl->link->link.prog); + if (prog_list_is_struct_ops(pl)) + WRITE_ONCE(item->kdata, bpf_struct_ops_map_kdata(pl->link->map)); + else + WRITE_ONCE(item->prog, pl->link->link.prog); } static u32 prog_list_id(struct bpf_prog_list *pl) { + if (prog_list_is_struct_ops(pl)) + return pl->link->map->id; return prog_list_prog(pl)->aux->id; } @@ -592,7 +669,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_cgroup_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } /** @@ -632,7 +709,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_cgroup_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i], i); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -687,7 +764,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_cgroup_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } continue; @@ -706,7 +783,7 @@ cleanup: css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_cgroup_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } @@ -945,7 +1022,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, old_pl_flags = pl->flags; bpf_cgroup_storages_assign(old_storage, pl->storage); } else { - pl = kmalloc_obj(*pl); + pl = kzalloc_obj(*pl); if (!pl) { bpf_cgroup_storages_free(new_storage); return -ENOMEM; @@ -1363,7 +1440,17 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; - if (type == BPF_LSM_CGROUP) { + if (type == BPF_STRUCT_OPS) { + u32 type_id = attr->query.type_id; + + atype = find_atype_by_struct_ops_id(type_id); + if (atype == CGROUP_BPF_ATTACH_TYPE_INVALID) + return -ENOENT; + from_atype = to_atype = atype; + flags = 0; + if (!cgroup_bpf_enabled(atype)) + goto skip_count; + } else if (type == BPF_LSM_CGROUP) { if (!effective_query && attr->query.prog_cnt && prog_ids && !prog_attach_flags) return -EINVAL; @@ -1389,6 +1476,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } } +skip_count: /* always output uattr->query.attach_flags as 0 during effective query */ flags = effective_query ? 0 : flags; if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) @@ -2846,6 +2934,274 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { const struct bpf_prog_ops cg_sockopt_prog_ops = { }; +static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *map; + struct cgroup *cgrp; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + cgroup_unlock(); + return 0; + } + + map = cg_link->map; + atype = bpf_struct_ops_map_cgroup_atype(map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) + break; + } + if (WARN_ON_ONCE(!pl)) { + cgroup_unlock(); + return -ENOENT; + } + + /* mark deleted so compute_effective_progs() skips it */ + pl->link = NULL; + if (update_effective_progs(cgrp, atype)) { + pl->link = cg_link; + purge_effective_progs(cgrp, pl, atype); + } + + hlist_del(&pl->node); + cgroup_struct_ops_link_detach_wake(cg_link, wake_poll); + cgrp->bpf.revisions[atype]++; + + kfree(pl); + static_branch_dec(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return 0; +} + +static int cgroup_struct_ops_link_detach(struct bpf_link *link) +{ + return __cgroup_struct_ops_link_detach(link, true); +} + +static void cgroup_struct_ops_link_dealloc(struct bpf_link *link) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + __cgroup_struct_ops_link_detach(link, false); + kfree(cg_link); +} + +static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) +{ + struct bpf_cgroup_link *cg_link = + container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return; + } + + seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); + seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); + cgroup_unlock(); +} + +static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return 0; + } + + info->struct_ops.map_id = cg_link->map->id; + info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); + cgroup_unlock(); + return 0; +} + +static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *new_map, + struct bpf_map *expected_old_map) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *old_map; + struct cgroup *cgrp; + bool found = false; + int err; + + if (!bpf_struct_ops_valid_to_reg(new_map)) + return -EINVAL; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + err = -ENOLINK; + goto out; + } + + old_map = cg_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) + goto out; + + atype = bpf_struct_ops_map_cgroup_atype(new_map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) { + found = true; + break; + } + } + if (!found) { + err = -ENOENT; + goto out; + } + + bpf_map_inc(new_map); + WRITE_ONCE(cg_link->map, new_map); + replace_effective_prog(cgrp, atype, pl); + bpf_map_put(old_map); + cgrp->bpf.revisions[atype]++; + +out: + cgroup_unlock(); + return err; +} + +static __poll_t cgroup_struct_ops_link_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_cgroup_link *link = file->private_data; + + poll_wait(file, &link->wait_hup, pts); + + return READ_ONCE(link->map) ? 0 : EPOLLHUP; +} + +static const struct bpf_link_ops cgroup_struct_ops_link_ops = { + .dealloc = cgroup_struct_ops_link_dealloc, + .detach = cgroup_struct_ops_link_detach, + .show_fdinfo = cgroup_struct_ops_link_show_fdinfo, + .fill_link_info = cgroup_struct_ops_link_fill_link_info, + .update_map = cgroup_struct_ops_link_update, + .poll = cgroup_struct_ops_link_poll, +}; + +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr) +{ + u32 flags = attr->link_create.flags; + u32 pl_flags = (flags & BPF_F_PREORDER) | BPF_F_ALLOW_MULTI; + enum cgroup_bpf_attach_type atype; + struct bpf_link_primer link_primer; + struct bpf_cgroup_link *link; + struct bpf_prog_list *pl = NULL; + struct hlist_head *progs; + struct cgroup *cgrp; + int err; + + if (flags & ~BPF_F_LINK_ATTACH_MASK) + return -EINVAL; + + /* + * Attaching struct_ops to cgroup is through link only. All relative + * position must be corresponding to a link id or fd. + */ + if (attr->link_create.cgroup.relative_fd && !(flags & BPF_F_LINK)) + return -EINVAL; + + link = kzalloc_obj(*link, GFP_USER); + if (!link) + return -ENOMEM; + + bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, + &cgroup_struct_ops_link_ops, NULL, + attr->link_create.attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) { + kfree(link); + return err; + } + + cgrp = cgroup_get_from_fd(attr->link_create.target_fd); + if (IS_ERR(cgrp)) { + err = PTR_ERR(cgrp); + goto cleanup; + } + + bpf_map_inc(map); + link->map = map; + link->cgroup = cgrp; + init_waitqueue_head(&link->wait_hup); + + atype = bpf_struct_ops_map_cgroup_atype(map); + progs = &cgrp->bpf.progs[atype]; + + cgroup_lock(); + + if (attr->link_create.cgroup.expected_revision && + attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { + err = -ESTALE; + goto unlock; + } + + if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) { + err = -E2BIG; + goto unlock; + } + + pl = kzalloc_obj(*pl); + if (!pl) { + err = -ENOMEM; + goto unlock; + } + + pl->link = link; + pl->flags = pl_flags; + cgrp->bpf.flags[atype] = BPF_F_ALLOW_MULTI; + + err = insert_pl_to_hlist(pl, progs, NULL, link, + flags | BPF_F_ALLOW_MULTI, attr->link_create.cgroup.relative_fd); + if (err) + goto unlock; + + err = update_effective_progs(cgrp, atype); + if (err) { + hlist_del(&pl->node); + goto unlock; + } + + cgrp->bpf.revisions[atype]++; + static_branch_inc(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return bpf_link_settle(&link_primer); + +unlock: + cgroup_unlock(); + +cleanup: + kfree(pl); + if (link->cgroup) { + cgroup_put(link->cgroup); + link->cgroup = NULL; + bpf_map_put(link->map); + link->map = NULL; + } + bpf_link_cleanup(&link_primer); + return err; +} + /* Common helpers for cgroup hooks. */ const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 01a1f1dd3b67..113486b15d29 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4758,6 +4758,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 732b35cc08d1..dabe01cd3def 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1756,7 +1756,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -6890,6 +6890,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; From 6d4ce907665d84bb27ca6fba218332a032426c4b Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 17 Sep 2026 13:05:36 -0700 Subject: [PATCH 164/660] bpf: Allow all struct_ops to use bpf_dynptr_from_skb() bpf_dynptr_from_skb() was only made available to bpf_qdisc, so far the only struct_ops type that needs to read an skb. The upcoming bpf_tcp_ops header-option hooks (parse_hdr/write_hdr_opt) also want to access the TCP options of an skb through a dynptr. All struct_ops programs share BPF_PROG_TYPE_STRUCT_OPS, so register bpf_kfunc_set_skb (which holds bpf_dynptr_from_skb) for that program type once, instead of per struct_ops. This makes bpf_dynptr_from_skb() available to bpf_tcp_ops and any future struct_ops. With the kfunc now provided to all of struct_ops, the bpf_qdisc-specific registration becomes redundant and is dropped: bpf_qdisc_kfunc_filter() only constrains kfuncs listed in qdisc_kfunc_ids, so removing bpf_dynptr_from_skb from that set (and from qdisc_common_kfunc_set) lets it fall through the filter unchanged, and bpf_qdisc keeps access via the generic struct_ops registration. Widening the registration is safe: a struct_ops that does not receive an skb in its context has nothing to pass to the helper. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-11-ameryhung@gmail.com --- net/core/filter.c | 1 + net/sched/bpf_qdisc.c | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index 61940e753552..47b7a8f73069 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -12887,6 +12887,7 @@ static int __init bpf_kfunc_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp); diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index 098ca02aed89..5691c13781a8 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c @@ -280,7 +280,6 @@ BTF_KFUNCS_START(qdisc_kfunc_ids) BTF_ID_FLAGS(func, bpf_skb_get_hash) BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE) BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_dynptr_from_skb) BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule) BTF_ID_FLAGS(func, bpf_qdisc_init_prologue) BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue) @@ -290,7 +289,6 @@ BTF_KFUNCS_END(qdisc_kfunc_ids) BTF_SET_START(qdisc_common_kfunc_set) BTF_ID(func, bpf_skb_get_hash) BTF_ID(func, bpf_kfree_skb) -BTF_ID(func, bpf_dynptr_from_skb) BTF_SET_END(qdisc_common_kfunc_set) BTF_SET_START(qdisc_enqueue_kfunc_set) From 5ac77ae329400929f944437df6f5791467738409 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 17 Sep 2026 13:05:37 -0700 Subject: [PATCH 165/660] bpf: tcp: Support selected sock_ops callbacks as struct_ops In LSFMMBPF 2025, I have talked about moving the BPF_PROG_TYPE_SOCK_OPS to a struct_ops interface [1]. The BPF_SOCK_OPS_*_CB enum interface has grown over time as new TCP callback points were added. A BPF_PROG_TYPE_SOCK_OPS program now commonly needs a large switch on sock_ops->op, and the shared bpf_sock_ops_kern context has become harder to extend because different callbacks have different locking, argument, skb, and helper requirements. The existing 'union { u32 args[4]; u32 replylong[4]; }' is also not reliable in passing args to bpf prog when there are multiple progs attached to a cgroup. The above has already been solved in struct_ops. Add a TCP-specific struct_ops type, bpf_tcp_ops, and support attaching it to cgroups. This allows each callback have its own func signature and allows the verifier to select kfuncs/helpers based on the specific struct_ops member being implemented. This patch wires up the following existing sock_ops callbacks: - BPF_SOCK_OPS_TIMEOUT_INIT - BPF_SOCK_OPS_RWND_INIT - BPF_SOCK_OPS_RTT_CB - BPF_SOCK_OPS_STATE_CB - BPF_SOCK_OPS_RETRANS_CB - BPF_SOCK_OPS_TCP_CONNECT_CB - BPF_SOCK_OPS_TCP_LISTEN_CB - BPF_SOCK_OPS_RTO_CB - BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB - BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB BASE_RTT is ignored as it is not particularly useful. NEEDS_ECN should be done in bpf-tcp-cc instead. The tstamp ones should be a separate struct_ops (e.g. "bpf_sock_ops") that can work in both TCP and UDP. timeout_init and rwnd_init could have a request_sock pointer. This patch tries a different API and directly passes the request_sock pointer as an arg. Two other approaches were considered before settling on having bpf_get_retval() read the dispatcher's run_ctx via saved_run_ctx. The first was to inherit the retval in the trampoline itself: add a helper in the four __bpf_prog_enter*() paths that, for struct_ops programs, copies the chained value from the caller's run_ctx (now saved_run_ctx) into the program's own run_ctx. It works but puts a per-enter program-type check on the generic trampoline fast path, taxing all fentry/fexit/lsm callers for a cgroup-struct_ops-only feature. The second was to do that same inherit only for the int-returning members via a gen_prologue that emits a hidden kfunc at the start of timeout_init/rwnd_init; this keeps the cost off the generic path and scoped to bpf_tcp_ops, but needs a kfunc + BTF_ID + prologue-emission machinery. The chosen approach avoids both: it touches neither the trampoline nor the program, since saved_run_ctx already points at the dispatcher's run_ctx that carries the value. [1], page 13: https://drive.google.com/file/d/1wjKZth6T0llLJ_ONPAL_6Q_jbxbAjByp/view?usp=sharing Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-12-ameryhung@gmail.com --- include/linux/bpf.h | 1 + include/net/tcp.h | 113 ++++++++++++++++++++++++- net/ipv4/Makefile | 1 + net/ipv4/af_inet.c | 1 + net/ipv4/bpf_tcp_ops.c | 188 +++++++++++++++++++++++++++++++++++++++++ net/ipv4/tcp.c | 1 + net/ipv4/tcp_input.c | 4 + net/ipv4/tcp_output.c | 2 + net/ipv4/tcp_timer.c | 1 + 9 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 net/ipv4/bpf_tcp_ops.c diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f34b410f903e..3abde9a2a375 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2634,6 +2634,7 @@ struct bpf_trace_run_ctx { struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; + int retval; struct bpf_run_ctx *saved_run_ctx; }; diff --git a/include/net/tcp.h b/include/net/tcp.h index 436495ff2271..af3747043778 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -2959,12 +2959,120 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) #endif +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF) + +struct bpf_tcp_ops { + /* Should return the initial SYN (active open) or SYN-ACK (passive open) + * retransmission timeout. Return the timeout in jiffies, or <= 0 for + * the kernel default. + * + * @req: request_sock on the passive (synack) path; NULL otherwise. + */ + int (*timeout_init)(struct sock *sk, struct request_sock *req); + + /* Should return the initial advertised receive window, in packets, + * or < 0 for the kernel default. @req as in timeout_init(). + */ + int (*rwnd_init)(struct sock *sk, struct request_sock *req); + + /* Called when an active connection becomes established. + * @skb is the SYNACK that completed the 3WHS, or NULL for a + * TCP_REPAIR socket (tcp_finish_connect() with no skb). + */ + void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable); + + /* Called when a passive connection becomes established. + * @skb is the ACK that completed the 3WHS. + */ + void (*passive_established)(struct sock *sk, struct sk_buff *skb); + + /* Called when the retransmission timer fires. */ + void (*rto)(struct sock *sk); + + /* Called on every RTT sample. + * @mrtt: the measured RTT, in microseconds. + * @srtt: the updated smoothed RTT. + */ + void (*rtt)(struct sock *sk, long mrtt, u32 srtt); + + /* Called when the connection changes TCP state. + * @state: the new state (one of the TCP_* states). + */ + void (*set_state)(struct sock *sk, int state); + + /* Called when an skb is retransmitted. + * @skb: the retransmitted skb. + * @err: tcp_transmit_skb() return value (0 on success). + */ + void (*retrans)(struct sock *sk, struct sk_buff *skb, int err); + + /* Called right before an active connection is initialized. */ + void (*connect)(struct sock *sk); + + /* Called on listen(2), right after the socket enters TCP_LISTEN. */ + void (*listen)(struct sock *sk); +}; + +#define bpf_tcp_ops_call(op, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct cgroup *cgrp; \ + \ + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ + rcu_read_lock_dont_migrate(); \ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + tcp_ops->op(sk, ##__VA_ARGS__); \ + } \ + rcu_read_unlock_migrate(); \ + } \ +} while (0) + +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ +({ \ + int __retval = (init_retval); \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct bpf_tramp_run_ctx run_ctx; \ + struct bpf_run_ctx *old_run_ctx; \ + struct sock *__sk = sk_to_full_sk(sk); \ + struct request_sock *req = NULL; \ + struct cgroup *cgrp; \ + \ + if (__sk) { \ + run_ctx.retval = (init_retval); \ + cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data); \ + if (!sk_fullsock(sk)) \ + req = (struct request_sock *)sk; \ + rcu_read_lock_dont_migrate(); \ + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \ + } \ + bpf_reset_run_ctx(old_run_ctx); \ + rcu_read_unlock_migrate(); \ + __retval = run_ctx.retval; \ + } \ + } \ + __retval; \ +}) +#else +#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) +#endif + static inline u32 tcp_timeout_init(struct sock *sk) { int timeout; timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); - + timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); if (timeout <= 0) timeout = TCP_TIMEOUT_INIT; return min_t(int, timeout, TCP_RTO_MAX); @@ -2975,7 +3083,7 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) int rwnd; rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); - + rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); if (rwnd < 0) rwnd = 0; return rwnd; @@ -2990,6 +3098,7 @@ static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + bpf_tcp_ops_call(rtt, sk, mrtt, srtt); } #if IS_ENABLED(CONFIG_SMC) diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index 06e21c26b76f..afbac63d1cb4 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -70,6 +70,7 @@ obj-$(CONFIG_TCP_AO) += tcp_ao.o ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o +obj-$(CONFIG_CGROUP_BPF) += bpf_tcp_ops.o endif ifdef CONFIG_GCOV_PROFILE_NETFILTER diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 32d006c1a8ee..ac8431da67f4 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -227,6 +227,7 @@ int __inet_listen_sk(struct sock *sk, int backlog) return err; tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); + bpf_tcp_ops_call(listen, sk); } return 0; } diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c new file mode 100644 index 000000000000..3febbc8dd1a0 --- /dev/null +++ b/net/ipv4/bpf_tcp_ops.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include + +static int timeout_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static int rwnd_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static void active_established_stub(struct sock *sk, struct sk_buff *skb__nullable) +{ +} + +static void passive_established_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void rto_stub(struct sock *sk) +{ +} + +static void rtt_stub(struct sock *sk, long mrtt, u32 srtt) +{ +} + +static void set_state_stub(struct sock *sk, int state) +{ +} + +static void retrans_stub(struct sock *sk, struct sk_buff *skb, int err) +{ +} + +static void connect_stub(struct sock *sk) +{ +} + +static void listen_stub(struct sock *sk) +{ +} + +static struct bpf_tcp_ops __bpf_tcp_ops = { + .timeout_init = timeout_init_stub, + .rwnd_init = rwnd_init_stub, + .active_established = active_established_stub, + .passive_established = passive_established_stub, + .rto = rto_stub, + .rtt = rtt_stub, + .set_state = set_state_stub, + .retrans = retrans_stub, + .connect = connect_stub, + .listen = listen_stub, +}; + +BPF_CALL_0(bpf_tcp_ops_get_retval) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + /* bpf_get_retval() is only exposed to timeout_init/rwnd_init, which + * always run via bpf_tcp_ops_call_int(). Its run_ctx carries the int + * return value chained across the bpf_tcp_ops attached to the cgroup + * and is this program's saved_run_ctx. + */ + if (WARN_ON_ONCE(!ctx->saved_run_ctx)) + return 0; + + return container_of(ctx->saved_run_ctx, struct bpf_tramp_run_ctx, + run_ctx)->retval; +} + +const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = { + .func = bpf_tcp_ops_get_retval, + .gpl_only = false, + .ret_type = RET_INTEGER, +}; + +static const struct bpf_func_proto * +get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + u32 moff = prog->aux->attach_st_ops_member_off; + + switch (func_id) { + case BPF_FUNC_sk_storage_get: + return &bpf_sk_storage_get_proto; + case BPF_FUNC_sk_storage_delete: + return &bpf_sk_storage_delete_proto; + case BPF_FUNC_setsockopt: + /* The listener is not locked. */ + if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || + moff == offsetof(struct bpf_tcp_ops, timeout_init)) + return NULL; + return &bpf_sk_setsockopt_proto; + case BPF_FUNC_getsockopt: + if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || + moff == offsetof(struct bpf_tcp_ops, timeout_init)) + return NULL; + return &bpf_sk_getsockopt_proto; + case BPF_FUNC_get_retval: + if (moff == offsetof(struct bpf_tcp_ops, timeout_init) || + moff == offsetof(struct bpf_tcp_ops, rwnd_init)) + return &bpf_tcp_ops_get_retval_proto; + return NULL; + default: + return bpf_base_func_proto(func_id, prog); + } +} + +static bool is_valid_access(int off, int size, enum bpf_access_type type, + const struct bpf_prog *prog, struct bpf_insn_access_aux *info) +{ + if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info)) + return false; + + if (base_type(info->reg_type) == PTR_TO_BTF_ID && + !bpf_type_has_unsafe_modifiers(info->reg_type) && + info->btf_id == btf_sock_ids[BTF_SOCK_TYPE_SOCK]) + /* promote it to tcp_sock */ + info->btf_id = btf_sock_ids[BTF_SOCK_TYPE_TCP]; + + return true; +} + +static int bpf_tcp_ops_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + return 0; +} + +static int bpf_tcp_ops_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + if (prog->sleepable) + return -EINVAL; + + return 0; +} + +static int bpf_tcp_ops_init(struct btf *btf) +{ + return 0; +} + +static int bpf_tcp_ops_validate(void *kdata) +{ + return 0; +} + +static const struct bpf_verifier_ops bpf_tcp_ops_verifier = { + .get_func_proto = get_func_proto, + .is_valid_access = is_valid_access, +}; + +static struct bpf_struct_ops bpf_tcp_ops = { + .verifier_ops = &bpf_tcp_ops_verifier, + .init_member = bpf_tcp_ops_init_member, + .check_member = bpf_tcp_ops_check_member, + .init = bpf_tcp_ops_init, + .validate = bpf_tcp_ops_validate, + .name = "bpf_tcp_ops", + .cgroup_atype = CGROUP_TCP_SOCK_OPS, + .cfi_stubs = &__bpf_tcp_ops, + .owner = THIS_MODULE, +}; + +static int __init __bpf_tcp_ops_init(void) +{ + return register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); +} +late_initcall(__bpf_tcp_ops_init); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 1c867a302444..a4456b419412 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2997,6 +2997,7 @@ void tcp_set_state(struct sock *sk, int state) if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + bpf_tcp_ops_call(set_state, sk, state); switch (state) { case TCP_ESTABLISHED: diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 0f60a1dbf927..69e6f3925073 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -6724,6 +6724,10 @@ void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb) tp->snd_cwnd_stamp = tcp_jiffies32; bpf_skops_established(sk, bpf_op, skb); + if (bpf_op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) + bpf_tcp_ops_call(active_established, sk, skb); + else + bpf_tcp_ops_call(passive_established, sk, skb); /* Initialize congestion control unless BPF initialized it already: */ if (!icsk->icsk_ca_initialized) tcp_init_congestion_control(sk); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index d960e3de7d50..f75a5a01d621 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -3678,6 +3678,7 @@ start: if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG)) tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB, TCP_SKB_CB(skb)->seq, segs, err); + bpf_tcp_ops_call(retrans, sk, skb, err); if (unlikely(err) && err != -EBUSY) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs); @@ -4302,6 +4303,7 @@ int tcp_connect(struct sock *sk) int err; tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL); + bpf_tcp_ops_call(connect, sk); #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO) /* Has to be checked late, after setting daddr/saddr/ops. diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index e56eae4bc341..3d49adc51766 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -290,6 +290,7 @@ static int tcp_write_timeout(struct sock *sk) tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB, icsk->icsk_retransmits, icsk->icsk_rto, (int)expired); + bpf_tcp_ops_call(rto, sk); if (expired) { /* Has it gone just too far? */ From 3bb54768fe3e61f85ca7666ce119628f2e251dde Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 17 Sep 2026 13:05:38 -0700 Subject: [PATCH 166/660] bpf: tcp: Support parse/len/write header option hooks in bpf_tcp_ops Add the TCP header option callbacks to the bpf_tcp_ops struct_ops type: parse_hdr - parse the options of an incoming skb on an established connection hdr_opt_len - reserve space in the TCP header for bpf options write_hdr_opt - write the reserved bpf options These mirror the BPF_SOCK_OPS_PARSE_HDR_OPT_CB, _HDR_OPT_LEN_CB and _WRITE_HDR_OPT_CB legacy sockops callbacks, but are exposed as struct_ops members so a program can implement them with normal function signatures and per-member helper sets. The reserved header window is shared between the legacy sockops and bpf_tcp_ops paths. tcp_{syn,synack,established}_options() first run the legacy BPF_SOCK_OPS_HDR_OPT_LEN_CB and then call hdr_opt_len, so both sources accumulate into opts->bpf_opt_len; at write time the legacy options are emitted first and bpf_tcp_ops writes after them. API design bpf_tcp_ops overloads the sock_ops header-option helpers rather than introducing a new API: bpf_reserve_hdr_opt(), bpf_store_hdr_opt() and bpf_load_hdr_opt() are exposed per-member (reserve for hdr_opt_len, store/load for write_hdr_opt, load for parse_hdr) and share the existing kernel option-walking core via _bpf_sock_ops{store,load}hdr_opt(), with the bpf_tcp_ops wrappers synthesizing a temporary bpf_sock_ops_kern from the program ctx. This keeps a port from the legacy BPF_SOCK_OPS*_HDR_OPT_CB callbacks mechanical (same helper calls) and adds no new UAPI helper/kfunc surface. An alternative considered was to drop the option helpers entirely: have hdr_opt_len reserve space purely through its return value, and introduce a dedicated TCP-header-option dynptr used for both reading and writing. That is a cleaner, more self-contained interface, but it is a larger change and does not reuse the legacy helpers, making a port from sockops less mechanical. It can be pursued as a follow-up; the helper-based interface here keeps this series focused on moving the hooks to struct_ops. The hdr_opt_len fast path in tcp_established_options() is gated by cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS). Note this is a global, per-attach-type static branch: it is enabled whenever any bpf_tcp_ops is attached, even one that does not implement hdr_opt_len or that is attached to a different cgroup. In those cases the block still runs but bpf_tcp_ops_hdr_opt_len() no-ops via the per-member check in the dispatch macro. A per-member/per-cgroup gate could be added later if the extra fast-path work proves measurable. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-13-ameryhung@gmail.com --- include/linux/filter.h | 5 ++ include/net/tcp.h | 43 ++++++++++ include/uapi/linux/bpf.h | 35 +++++--- net/core/filter.c | 32 +++++--- net/ipv4/bpf_tcp_ops.c | 144 ++++++++++++++++++++++++++++++++- net/ipv4/tcp_input.c | 13 +++ net/ipv4/tcp_output.c | 102 +++++++++++++++++------ tools/include/uapi/linux/bpf.h | 35 +++++--- 8 files changed, 344 insertions(+), 65 deletions(-) diff --git a/include/linux/filter.h b/include/linux/filter.h index b17222db2efc..422284b4fa96 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1930,6 +1930,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde return XDP_REDIRECT; } +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags); +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags); + #ifdef CONFIG_NET int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, diff --git a/include/net/tcp.h b/include/net/tcp.h index af3747043778..d61ee00052e3 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -3011,6 +3011,48 @@ struct bpf_tcp_ops { /* Called on listen(2), right after the socket enters TCP_LISTEN. */ void (*listen)(struct sock *sk); + + /* + * Parse the TCP header options of an incoming skb received on an + * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() + * to access the options. + */ + void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); + + /* + * Reserve space in the outgoing TCP header for options to be written + * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. + * + * @skb: outgoing packet. NULL when called from tcp_current_mss() + * (MSS sizing). + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @remaining: pointer to the size of space still available; cast it + * using bpf_rdonly_cast() before dereferencing. + */ + void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + unsigned int *remaining); + + /* + * Write header options into the space reserved earlier by hdr_opt_len(). + * Use bpf_store_hdr_opt() to write; it appends within the reserved window + * shared with legacy SOCKOPS. + * + * @skb: outgoing packet. + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @opt_off: offset in the outgoing @skb's TCP header where the + * bpf_tcp_ops portion of the reserved window begins, i.e. after + * the kernel and legacy options. + */ + void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + u32 opt_off); }; #define bpf_tcp_ops_call(op, sk, ...) \ @@ -3062,6 +3104,7 @@ do { \ } \ __retval; \ }) + #else #define bpf_tcp_ops_call(op, sk, ...) do { } while (0) #define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index dabe01cd3def..6330b7d745c5 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4867,15 +4867,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4911,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4933,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +4951,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +4968,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +4980,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +4992,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description diff --git a/net/core/filter.c b/net/core/filter.c index 47b7a8f73069..5feb99884682 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8053,17 +8053,14 @@ static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend, return ERR_PTR(-ENOMSG); } -BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - void *, search_res, u32, len, u64, flags) +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags) { bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN; const u8 *op, *opend, *magic, *search = search_res; u8 search_kind, search_len, copy_len, magic_len; int ret; - if (!is_locked_tcp_sock_ops(bpf_sock)) - return -EOPNOTSUPP; - /* 2 byte is the minimal option len except TCPOPT_NOP and * TCPOPT_EOL which are useless for the bpf prog to learn * and this helper disallow loading them also. @@ -8124,6 +8121,15 @@ BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return ret; } +BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + void *, search_res, u32, len, u64, flags) +{ + if (!is_locked_tcp_sock_ops(bpf_sock)) + return -EOPNOTSUPP; + + return __bpf_sock_ops_load_hdr_opt(bpf_sock, search_res, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .func = bpf_sock_ops_load_hdr_opt, .gpl_only = false, @@ -8134,17 +8140,14 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .arg4_type = ARG_ANYTHING, }; -BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - const void *, from, u32, len, u64, flags) +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags) { u8 new_kind, new_kind_len, magic_len = 0, *opend; const u8 *op, *new_op, *magic = NULL; struct sk_buff *skb; bool eol; - if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) - return -EPERM; - if (len < 2 || flags) return -EINVAL; @@ -8202,6 +8205,15 @@ BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return 0; } +BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + const void *, from, u32, len, u64, flags) +{ + if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) + return -EPERM; + + return __bpf_sock_ops_store_hdr_opt(bpf_sock, from, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .func = bpf_sock_ops_store_hdr_opt, .gpl_only = false, diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index 3febbc8dd1a0..681fed642999 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,26 @@ static void listen_stub(struct sock *sk) { } +static void parse_hdr_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void hdr_opt_len_stub(struct sock *sk, struct sk_buff *skb__nullable, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + unsigned int *remaining) +{ +} + +static void write_hdr_opt_stub(struct sock *sk, struct sk_buff *skb, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + u32 opt_off) +{ +} + static struct bpf_tcp_ops __bpf_tcp_ops = { .timeout_init = timeout_init_stub, .rwnd_init = rwnd_init_stub, @@ -66,6 +87,104 @@ static struct bpf_tcp_ops __bpf_tcp_ops = { .retrans = retrans_stub, .connect = connect_stub, .listen = listen_stub, + .parse_hdr = parse_hdr_stub, + .hdr_opt_len = hdr_opt_len_stub, + .write_hdr_opt = write_hdr_opt_stub, +}; + +BPF_CALL_4(bpf_tcp_ops_store_hdr_opt, void *, ctx, const void *, from, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + u32 opt_off = args[5]; + u8 *op, *opend; + + /* + * bpf_tcp_ops does not keep track of the end of the written TCP header + * options, so search for it every time the helper is called. The free + * space is NOP-filled, so a TCPOPT_NOP ends the search rather than being + * skipped as in a normal option walk in sockops. + */ + op = skb->data + opt_off; + opend = skb->data + tcp_hdrlen(skb); + while (op < opend && *op != TCPOPT_NOP) { + if (*op == TCPOPT_EOL || op + 1 >= opend || op[1] < 2) + break; + op += op[1]; + } + + sock_ops.skb = skb; + sock_ops.skb_data_end = op; + sock_ops.remaining_opt_len = opend - op; + + return __bpf_sock_ops_store_hdr_opt(&sock_ops, from, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_store_hdr_opt_proto = { + .func = bpf_tcp_ops_store_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_4(bpf_tcp_ops_load_hdr_opt, void *, ctx, void *, search_res, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + + /* + * No flags supported. In particular BPF_LOAD_HDR_OPT_TCP_SYN, which + * loads from the saved SYN, is not available because bpf_tcp_ops has no + * carrier to track the SYN source across the hooks. + */ + if (flags) + return -EINVAL; + + sock_ops.skb = skb; + sock_ops.skb_data_end = skb->data + tcp_hdrlen(skb); + + return __bpf_sock_ops_load_hdr_opt(&sock_ops, search_res, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_load_hdr_opt_proto = { + .func = bpf_tcp_ops_load_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_3(bpf_tcp_ops_reserve_hdr_opt, void *, ctx, u32, len, u64, flags) +{ + u64 *args = ctx; + unsigned int *remaining = (void *)(unsigned long)args[5]; + + if (flags || len < 2) + return -EINVAL; + + if (len > *remaining) + return -ENOSPC; + + *remaining -= len; + return 0; +} + +static const struct bpf_func_proto bpf_tcp_ops_reserve_hdr_opt_proto = { + .func = bpf_tcp_ops_reserve_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, }; BPF_CALL_0(bpf_tcp_ops_get_retval) @@ -102,14 +221,20 @@ get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_storage_delete: return &bpf_sk_storage_delete_proto; case BPF_FUNC_setsockopt: - /* The listener is not locked. */ + /* The sk may be an unlocked listener (synack path) or NULL + * fullsock; disable for members that can run unlocked. + */ if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || - moff == offsetof(struct bpf_tcp_ops, timeout_init)) + moff == offsetof(struct bpf_tcp_ops, timeout_init) || + moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) || + moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) return NULL; return &bpf_sk_setsockopt_proto; case BPF_FUNC_getsockopt: if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || - moff == offsetof(struct bpf_tcp_ops, timeout_init)) + moff == offsetof(struct bpf_tcp_ops, timeout_init) || + moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) || + moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) return NULL; return &bpf_sk_getsockopt_proto; case BPF_FUNC_get_retval: @@ -117,6 +242,19 @@ get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) moff == offsetof(struct bpf_tcp_ops, rwnd_init)) return &bpf_tcp_ops_get_retval_proto; return NULL; + case BPF_FUNC_reserve_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, hdr_opt_len)) + return &bpf_tcp_ops_reserve_hdr_opt_proto; + return NULL; + case BPF_FUNC_load_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, parse_hdr) || + moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_load_hdr_opt_proto; + return NULL; + case BPF_FUNC_store_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_store_hdr_opt_proto; + return NULL; default: return bpf_base_func_proto(func_id, prog); } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 69e6f3925073..6ac6f9d5b6c3 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -208,6 +208,18 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, } #endif +static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb) +{ + switch (sk->sk_state) { + case TCP_SYN_RECV: + case TCP_SYN_SENT: + case TCP_LISTEN: + return; + } + + bpf_tcp_ops_call(parse_hdr, sk, skb); +} + static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb, unsigned int len) { @@ -6461,6 +6473,7 @@ syn_challenge: pass: bpf_skops_parse_hdr(sk, skb); + bpf_tcp_ops_parse_hdr(sk, skb); return true; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index f75a5a01d621..908944d409d6 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -536,43 +536,53 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, enum tcp_synack_type synack_type, struct tcp_out_options *opts) { - u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len; - struct bpf_sock_ops_kern sock_ops; - int err; + u8 first_opt_off, nr_written = 0, max_opt_len = opts->bpf_opt_len; if (likely(!max_opt_len)) return; - memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); - - sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; - - if (req) { - sock_ops.sk = (struct sock *)req; - sock_ops.syn_skb = syn_skb; - } else { - sock_owned_by_me(sk); - - sock_ops.is_fullsock = 1; - sock_ops.is_locked_tcp_sock = 1; - sock_ops.sk = sk; - } - - sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); - sock_ops.remaining_opt_len = max_opt_len; first_opt_off = tcp_hdrlen(skb) - max_opt_len; - bpf_skops_init_skb(&sock_ops, skb, first_opt_off); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) { + struct bpf_sock_ops_kern sock_ops; + int err; - if (err) - nr_written = 0; - else - nr_written = max_opt_len - sock_ops.remaining_opt_len; + memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); + + sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; + + if (req) { + sock_ops.sk = (struct sock *)req; + sock_ops.syn_skb = syn_skb; + } else { + sock_owned_by_me(sk); + + sock_ops.is_fullsock = 1; + sock_ops.is_locked_tcp_sock = 1; + sock_ops.sk = sk; + } + + sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); + sock_ops.remaining_opt_len = max_opt_len; + bpf_skops_init_skb(&sock_ops, skb, first_opt_off); + + err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (!err) + nr_written = max_opt_len - sock_ops.remaining_opt_len; + } if (nr_written < max_opt_len) memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP, max_opt_len - nr_written); + + /* + * bpf_tcp_ops portion is NOP-filled (everything past the sockops + * writer's bytes). The writer finds the append point by scanning from + * first_opt_off + nr_written to the first NOP. + */ + bpf_tcp_ops_call(write_hdr_opt, sk, skb, req, syn_skb, synack_type, + first_opt_off + nr_written); } #else static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, @@ -594,6 +604,32 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, } #endif +static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + unsigned int remaining_out = remaining, reserved; + + if (!remaining) + return 0; + + /* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */ + bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out); + + reserved = remaining - remaining_out; + if (!reserved) + return remaining; + + /* round up to 4 bytes */ + reserved = (reserved + 3) & ~3; + + opts->bpf_opt_len += reserved; + return remaining - reserved; +} + static __be32 *process_tcp_ao_options(struct tcp_sock *tp, const struct tcp_request_sock *tcprsk, struct tcp_out_options *opts, @@ -1053,6 +1089,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, remaining); + remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1141,6 +1179,8 @@ static unsigned int tcp_synack_options(const struct sock *sk, remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, synack_type, opts, remaining); + remaining = bpf_tcp_ops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, + synack_type, opts, remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1157,6 +1197,7 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb unsigned int eff_sacks; opts->options = 0; + opts->bpf_opt_len = 0; /* Better than switch (key.type) as it has static branches */ if (tcp_key_is_md5(key)) { @@ -1244,6 +1285,15 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb size = MAX_TCP_OPTION_SPACE - remaining; } + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { + unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + + remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + remaining); + + size = MAX_TCP_OPTION_SPACE - remaining; + } + return size; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index dabe01cd3def..6330b7d745c5 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4867,15 +4867,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4911,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4933,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +4951,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +4968,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +4980,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +4992,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description From 6bad848e3f70b9e7c0d242e8a8a37ad2b1e807ac Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:39 -0700 Subject: [PATCH 167/660] libbpf: Support attaching struct_ops to a cgroup Add bpf_map__attach_cgroup_opts() to attach a struct_ops map to a cgroup through a BPF link. Also extend struct bpf_prog_query_opts with a type_id field so a BPF_STRUCT_OPS query on a cgroup can select the struct_ops type to enumerate. Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-14-ameryhung@gmail.com --- tools/lib/bpf/bpf.c | 2 ++ tools/lib/bpf/bpf.h | 3 +- tools/lib/bpf/libbpf.c | 64 ++++++++++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 3 ++ tools/lib/bpf/libbpf.map | 1 + 5 files changed, 72 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 96819c082c77..a9de7f107cf7 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -934,6 +934,7 @@ int bpf_link_create(int prog_fd, int target_fd, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: relative_fd = OPTS_GET(opts, cgroup.relative_fd, 0); relative_id = OPTS_GET(opts, cgroup.relative_id, 0); if (relative_fd && relative_id) @@ -1056,6 +1057,7 @@ int bpf_prog_query_opts(int target, enum bpf_attach_type type, attr.query.attach_type = type; attr.query.query_flags = OPTS_GET(opts, query_flags, 0); attr.query.count = OPTS_GET(opts, count, 0); + attr.query.type_id = OPTS_GET(opts, type_id, 0); attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); attr.query.link_ids = ptr_to_u64(OPTS_GET(opts, link_ids, NULL)); attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 7534a593edae..490e8cb4ba53 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -637,9 +637,10 @@ struct bpf_prog_query_opts { __u32 *link_ids; __u32 *link_attach_flags; __u64 revision; + __u32 type_id; size_t :0; }; -#define bpf_prog_query_opts__last_field revision +#define bpf_prog_query_opts__last_field type_id /** * @brief **bpf_prog_query_opts()** queries the BPF programs and BPF links diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 33ff61b151f3..2f53afc985cf 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -14174,6 +14174,70 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) return &link->link; } +struct bpf_link *bpf_map__attach_cgroup_opts(const struct bpf_map *map, int cgroup_fd, + const struct bpf_cgroup_opts *opts) +{ + LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); + struct bpf_link_struct_ops *link; + __u32 relative_id, zero = 0; + int err, fd, relative_fd; + + if (!OPTS_VALID(opts, bpf_cgroup_opts)) + return libbpf_err_ptr(-EINVAL); + + if (!bpf_map__is_struct_ops(map)) { + pr_warn("map '%s': can't attach non-struct_ops map\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + if (map->fd < 0) { + pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + if (!(map->def.map_flags & BPF_F_LINK)) { + pr_warn("map '%s': can't attach to cgroup without BPF_F_LINK\n", map->name); + return libbpf_err_ptr(-EINVAL); + } + + relative_id = OPTS_GET(opts, relative_id, 0); + relative_fd = OPTS_GET(opts, relative_fd, 0); + + if (relative_fd && relative_id) { + pr_warn("map '%s': relative_fd and relative_id cannot be set at the same time\n", + map->name); + return libbpf_err_ptr(-EINVAL); + } + + link_create_opts.cgroup.expected_revision = OPTS_GET(opts, expected_revision, 0); + link_create_opts.cgroup.relative_fd = relative_fd; + link_create_opts.cgroup.relative_id = relative_id; + link_create_opts.flags = OPTS_GET(opts, flags, 0); + + link = calloc(1, sizeof(*link)); + if (!link) + return libbpf_err_ptr(-ENOMEM); + + err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); + if (err && err != -EBUSY) { + free(link); + return libbpf_err_ptr(err); + } + + link->link.detach = bpf_link__detach_struct_ops; + + fd = bpf_link_create(map->fd, cgroup_fd, BPF_STRUCT_OPS, &link_create_opts); + if (fd < 0) { + free(link); + return libbpf_err_ptr(fd); + } + + link->link.fd = fd; + link->map_fd = map->fd; + + return &link->link; +} + /* * Swap the back struct_ops of a link with a new struct_ops map. */ diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 48266e752223..026932f20962 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -960,6 +960,9 @@ bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd, struct bpf_map; LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map); +LIBBPF_API struct bpf_link *bpf_map__attach_cgroup_opts(const struct bpf_map *map, + int cgroup_fd, + const struct bpf_cgroup_opts *opts); LIBBPF_API int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map); struct bpf_iter_attach_opts { diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index a811a1b3a085..7a84dd00ce95 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -458,6 +458,7 @@ LIBBPF_1.7.0 { LIBBPF_1.8.0 { global: + bpf_map__attach_cgroup_opts; bpf_program__add_flags; bpf_program__attach_tracing_multi; bpf_program__clear_flags; From ae7b5234fa8b4150caae67844507c8d765731417 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 17 Sep 2026 13:05:40 -0700 Subject: [PATCH 168/660] selftests/bpf: Test attaching struct_ops to a cgroup Exercise attaching the bpf_tcp_ops struct_ops to cgroups via the generic cgroup link infrastructure. The struct_ops instances record their execution order and the previous return value to validate correctness. Subtests: - query: BPF_F_QUERY_EFFECTIVE and attached query return the maps - order: BPF_F_PREORDER vs attach order within a cgroup - before_after: BPF_F_BEFORE/BPF_F_AFTER relative positioning - update: bpf_link__update_map swaps a link's map, keeping its slot - retval: int return value chained across timeout_init progs of multiple bpf_tcp_ops attached to a cgroup - hierarchy: parent and child attachments merge in the child's effective array (descendant before ancestor) - inherit: a child created after the attach inherits the parent's prog Signed-off-by: Martin KaFai Lau Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-15-ameryhung@gmail.com --- .../selftests/bpf/prog_tests/bpf_tcp_ops.c | 560 ++++++++++++++++++ .../testing/selftests/bpf/progs/bpf_tcp_ops.c | 141 +++++ 2 files changed, 701 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c create mode 100644 tools/testing/selftests/bpf/progs/bpf_tcp_ops.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c new file mode 100644 index 000000000000..6435ed1c2cf6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "cgroup_helpers.h" +#include "bpf_tcp_ops.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops" +#define TEST_NETNS "bpf_tcp_ops" + +static __s32 get_bpf_tcp_ops_type_id(void) +{ + struct btf *vmlinux_btf; + __s32 type_id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "load_vmlinux_btf")) + return -1; + + type_id = btf__find_by_name_kind(vmlinux_btf, "bpf_tcp_ops", BTF_KIND_STRUCT); + btf__free(vmlinux_btf); + + ASSERT_GT(type_id, 0, "find_bpf_tcp_ops"); + return type_id; +} + +static void reset_order(struct bpf_tcp_ops *skel) +{ + memset(skel->bss->listen_order, 0, sizeof(skel->bss->listen_order)); + memset(skel->bss->connect_order, 0, sizeof(skel->bss->connect_order)); + skel->bss->listen_cnt = 0; + skel->bss->connect_cnt = 0; +} + +static void do_listen_connect(int family) +{ + const char *addr = family == AF_INET ? "127.0.0.1" : "::1"; + int server_fd, client_fd; + + server_fd = start_server(family, SOCK_STREAM, addr, 0, 0); + if (!ASSERT_GE(server_fd, 0, "start_server")) + return; + + client_fd = connect_to_fd(server_fd, 0); + if (ASSERT_OK_FD(client_fd, "connect_to_fd")) + close(client_fd); + + close(server_fd); +} + +/* + * Attach ops1 and ops2 normally (in that order), then ops3 with + * BPF_F_PREORDER. Expected execution order: [3, 1, 2] — ops3 runs + * first despite being attached last, ops1 before ops2 by attach order. + */ +static void test_order(int cgroup_fd, struct bpf_tcp_ops *skel, int family) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(family); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 3, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 2, "listen_order[2]"); + + ASSERT_EQ(skel->bss->connect_cnt, 3, "connect_cnt"); + ASSERT_EQ(skel->bss->connect_order[0], 3, "connect_order[0]"); + ASSERT_EQ(skel->bss->connect_order[1], 1, "connect_order[1]"); + ASSERT_EQ(skel->bss->connect_order[2], 2, "connect_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_order_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_order(cgroup_fd, skel, AF_INET); + test_order(cgroup_fd, skel, AF_INET6); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Position a new attachment relative to an existing one. Attach ops1, then + * ops2 with BPF_F_BEFORE ops1, then ops3 with BPF_F_AFTER ops2. Expected + * execution order: [2, 3, 1]. For struct_ops, relative_fd refers to a link + * fd, so BPF_F_LINK must be set. + */ +static void test_before_after(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, opts); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + opts.flags = BPF_F_BEFORE | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link1); + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link2, "attach_ops2_before")) + goto done; + + opts.flags = BPF_F_AFTER | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link2); + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_after")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 1, "listen_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_before_after_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_before_after(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +static void test_query(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_prog_query_opts, query_opts); + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + struct bpf_link *link1 = NULL, *link2 = NULL; + __u32 map1_id, map2_id, map_ids[2] = {}; + __s32 type_id; + + type_id = get_bpf_tcp_ops_type_id(); + if (type_id <= 0) + return; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops1), &info, &info_len); + map1_id = info.id; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops2), &info, &info_len); + map2_id = info.id; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + /* query effective: expect 2 entries in attachment order */ + query_opts.type_id = type_id; + query_opts.prog_ids = map_ids; + query_opts.count = ARRAY_SIZE(map_ids); + query_opts.query_flags = BPF_F_QUERY_EFFECTIVE; + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_effective"); + ASSERT_EQ(query_opts.count, 2, "query_effective_count"); + ASSERT_EQ(map_ids[0], map1_id, "map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "map_ids[1]"); + + /* query attached (non-effective): expect 2 entries */ + memset(map_ids, 0, sizeof(map_ids)); + query_opts.query_flags = 0; + query_opts.count = ARRAY_SIZE(map_ids); + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_attached"); + ASSERT_EQ(query_opts.count, 2, "query_attached_count"); + ASSERT_EQ(map_ids[0], map1_id, "attached_map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "attached_map_ids[1]"); + +done: + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_query_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_query(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* Must match progs/bpf_tcp_ops.c */ +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 + +/* + * Attach three struct_ops implementing timeout_init to the same cgroup; they + * run in attach order [retval1, retval2, retval3]. timeout_init's return value + * is chained: the first prog reads the kernel seed via bpf_get_retval() (0, + * since no legacy sockops prog is attached) and returns OPS_RETVAL1; each + * subsequent prog must then observe the previous prog's return value. This + * proves the trampoline inherits the retval across an array of struct_ops. + */ +static void test_retval(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + skel->bss->retval_saw1 = -1; + skel->bss->retval_saw2 = -1; + skel->bss->retval_saw3 = -1; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_retval1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_retval2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval3, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link3, "attach_retval3")) + goto done; + + do_listen_connect(AF_INET6); + + /* First prog inherits the kernel seed (no legacy sockops -> 0). */ + ASSERT_EQ(skel->bss->retval_saw1, 0, "retval_saw1"); + /* Each subsequent prog inherits the previous prog's return value. */ + ASSERT_EQ(skel->bss->retval_saw2, OPS_RETVAL1, "retval_saw2"); + ASSERT_EQ(skel->bss->retval_saw3, OPS_RETVAL2, "retval_saw3"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_retval_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_retval(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * bpf_link__update_map() swaps the struct_ops map backing an attached link. + * The link keeps its position, including BPF_F_PREORDER, across the update. + * Attach ops1 (normal) and ops2 (preorder): order [2, 1]. Update the normal + * link to ops3 -> [2, 3]; update the preorder link to ops1 -> [1, 3]. + */ +static void test_update(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link = NULL, *link_pre = NULL; + + link = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_ops1")) + goto done; + + link_pre = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link_pre, "attach_ops2_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_cnt, 2, "cnt_initial"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_initial"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "order1_initial"); + + /* Update the normal link's map (ops1 -> ops3); position is unchanged. */ + if (!ASSERT_OK(bpf_link__update_map(link, skel->maps.tcp_ops3), "update_normal")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_after_normal"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_normal"); + + /* Update the preorder link's map (ops2 -> ops1); it stays first. */ + if (!ASSERT_OK(bpf_link__update_map(link_pre, skel->maps.tcp_ops1), "update_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 1, "order0_after_preorder"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_preorder"); + +done: + bpf_link__destroy(link_pre); + bpf_link__destroy(link); +} + +static void run_update_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_update(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Two-level hierarchy. Attach ops1 to the parent and ops2 to the child, then + * trigger from a socket in the child. Descendant progs run before ancestor + * progs, so the order is [2 (child), 1 (parent)]. + */ +static void test_hierarchy(int parent_fd, int child_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL, *clink = NULL; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + clink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, child_fd, NULL); + if (!ASSERT_OK_PTR(clink, "attach_child")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 2, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + +done: + bpf_link__destroy(clink); + bpf_link__destroy(plink); +} + +static void run_hierarchy_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd, child_fd = -1; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_hierarchy(parent_fd, child_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + close(parent_fd); +} + +/* + * Attach ops1 to the parent, then create and join the child cgroup. The child + * is created after the attach, so it must inherit the parent's effective progs + * via cgroup_bpf_inherit(). A socket in the child runs the parent's prog. + */ +static void test_inherit(int parent_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL; + int child_fd = -1; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 1, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 1, "listen_order[0]"); + +done: + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + bpf_link__destroy(plink); +} + +static void run_inherit_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_inherit(parent_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(parent_fd); +} + +void test_bpf_tcp_ops(void) +{ + if (test__start_subtest("order")) + run_order_subtest(); + if (test__start_subtest("before_after")) + run_before_after_subtest(); + if (test__start_subtest("query")) + run_query_subtest(); + if (test__start_subtest("retval")) + run_retval_subtest(); + if (test__start_subtest("update")) + run_update_subtest(); + if (test__start_subtest("hierarchy")) + run_hierarchy_subtest(); + if (test__start_subtest("inherit")) + run_inherit_subtest(); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c new file mode 100644 index 000000000000..94a7f52573d5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include + +#define MAX_CGROUP_OPS 8 + +/* Call order for listen and connect, indexed by call sequence */ +u32 listen_order[MAX_CGROUP_OPS]; +u32 listen_cnt; + +u32 connect_order[MAX_CGROUP_OPS]; +u32 connect_cnt; + +static void record_listen(int id) +{ + u32 idx = listen_cnt; + + if (idx < MAX_CGROUP_OPS) { + listen_order[idx] = id; + listen_cnt = idx + 1; + } +} + +static void record_connect(int id) +{ + u32 idx = connect_cnt; + + if (idx < MAX_CGROUP_OPS) { + connect_order[idx] = id; + connect_cnt = idx + 1; + } +} + +/* struct_ops instance 1 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_listen, struct sock *sk) +{ + record_listen(1); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_connect, struct sock *sk) +{ + record_connect(1); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops1 = { + .listen = (void *)tcp_ops1_listen, + .connect = (void *)tcp_ops1_connect, +}; + +/* struct_ops instance 2 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_listen, struct sock *sk) +{ + record_listen(2); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_connect, struct sock *sk) +{ + record_connect(2); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops2 = { + .listen = (void *)tcp_ops2_listen, + .connect = (void *)tcp_ops2_connect, +}; + +/* struct_ops instance 3 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_listen, struct sock *sk) +{ + record_listen(3); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_connect, struct sock *sk) +{ + record_connect(3); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops3 = { + .listen = (void *)tcp_ops3_listen, + .connect = (void *)tcp_ops3_connect, +}; + +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 +#define OPS_RETVAL3 33 + +int retval_saw1; +int retval_saw2; +int retval_saw3; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval1_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw1 = bpf_get_retval(); + return OPS_RETVAL1; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval1 = { + .timeout_init = (void *)tcp_ops_retval1_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval2_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw2 = bpf_get_retval(); + return OPS_RETVAL2; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval2 = { + .timeout_init = (void *)tcp_ops_retval2_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval3_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw3 = bpf_get_retval(); + return OPS_RETVAL3; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval3 = { + .timeout_init = (void *)tcp_ops_retval3_timeout_init, +}; + +char _license[] SEC("license") = "GPL"; From d7399d91417bc96b20862bea664dfd00821d9445 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 17 Sep 2026 13:05:41 -0700 Subject: [PATCH 169/660] selftests/bpf: Add test for bpf_tcp_ops header option hooks Add a test exercising the bpf_tcp_ops parse_hdr, hdr_opt_len and write_hdr_opt members together with the header option helpers. The struct_ops program (progs/bpf_tcp_ops_hdr.c) reserves space in hdr_opt_len via bpf_reserve_hdr_opt(), writes an experimental option in write_hdr_opt via bpf_store_hdr_opt(), and recovers it in parse_hdr via bpf_load_hdr_opt() on the incoming skb. Each hook bumps a counter and the parse hook records the option payload, so the three callbacks and all three overloaded helpers are covered. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260917200542.3689605-16-ameryhung@gmail.com --- .../bpf/prog_tests/bpf_tcp_ops_hdr.c | 77 +++++++++++++++++ .../selftests/bpf/progs/bpf_tcp_ops_hdr.c | 86 +++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c create mode 100644 tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c new file mode 100644 index 000000000000..244605e69e40 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "cgroup_helpers.h" +#include "bpf_tcp_ops_hdr.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops_hdr" +#define TEST_NETNS "bpf_tcp_ops_hdr" + +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +static void run_hdr_opt(void) +{ + struct bpf_tcp_ops_hdr *skel = NULL; + struct bpf_link *link = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd, lfd = -1, fd = -1; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops_hdr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + link = bpf_map__attach_cgroup_opts(skel->maps.test_hdr_ops, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_cgroup")) + goto done; + + /* + * One direction of data is enough to exercise all hooks: both peers + * share the cgroup struct_ops, so the sender runs hdr_opt_len/write + * and the receiver runs parse. + */ + lfd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (!ASSERT_GE(lfd, 0, "start_server")) + goto done; + + fd = connect_to_fd(lfd, 0); + if (!ASSERT_OK_FD(fd, "connect_to_fd")) + goto done; + + if (!ASSERT_OK(send_recv_data(lfd, fd, 64), "send_recv_data")) + goto done; + + /* Reserve + write hooks ran while sending. */ + ASSERT_GT(skel->bss->hdr_opt_len_cnt, 0, "hdr_opt_len_cnt"); + ASSERT_GT(skel->bss->write_cnt, 0, "write_cnt"); + /* Parse hook ran and recovered our option on the receive side. */ + ASSERT_GT(skel->bss->parse_cnt, 0, "parse_cnt"); + ASSERT_GT(skel->bss->found_cnt, 0, "found_cnt"); + ASSERT_EQ(skel->bss->found_d0, TEST_OPT_D0, "found_d0"); + ASSERT_EQ(skel->bss->found_d1, TEST_OPT_D1, "found_d1"); + +done: + if (fd >= 0) + close(fd); + if (lfd >= 0) + close(lfd); + bpf_link__destroy(link); + bpf_tcp_ops_hdr__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +void test_bpf_tcp_ops_hdr(void) +{ + run_hdr_opt(); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c new file mode 100644 index 000000000000..f3e3dff13784 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include +#include + +/* Experimental option kind and payload written/parsed by this test. */ +#define TEST_OPT_KIND 0xFD +#define TEST_OPT_LEN 4 +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +int hdr_opt_len_cnt; +int write_cnt; +int parse_cnt; +int found_cnt; +__u8 found_d0; +__u8 found_d1; + +SEC("struct_ops") +void BPF_PROG(test_hdr_opt_len, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, unsigned int *remaining) +{ + hdr_opt_len_cnt++; + + /* + * Reserve TEST_OPT_LEN bytes; the helper decrements *remaining. Stacks + * with other progs in the cgroup hierarchy. + */ + bpf_reserve_hdr_opt(ctx, TEST_OPT_LEN, 0); +} + +SEC("struct_ops") +void BPF_PROG(test_write_hdr_opt, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, __u32 opt_off) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + /* + * bpf_store_hdr_opt() takes the program ctx (the kernel reads the + * outgoing skb from it); it appends after any options already written + * in the reserved window, rejects duplicates, and confines the write to + * the header option scratch. Stacks across progs in the cgroup hierarchy. + */ + if (bpf_store_hdr_opt(ctx, opt, sizeof(opt), 0)) + return; + + write_cnt++; +} + +SEC("struct_ops") +void BPF_PROG(test_parse_hdr, struct sock *sk, struct sk_buff *skb) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + parse_cnt++; + + /* + * Look up the experimental option written by test_write_hdr_opt() in + * the incoming skb. For an experimental kind the search matches on the + * 2-byte magic in opt[2..3]; on a match the found option is copied back + * into opt[]. + */ + if (bpf_load_hdr_opt(ctx, opt, sizeof(opt), 0) < 0) + return; + + found_d0 = opt[2]; + found_d1 = opt[3]; + found_cnt++; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops test_hdr_ops = { + .hdr_opt_len = (void *)test_hdr_opt_len, + .write_hdr_opt = (void *)test_write_hdr_opt, + .parse_hdr = (void *)test_parse_hdr, +}; + +char _license[] SEC("license") = "GPL"; From 559b90b855ce535572e743ed79a59f728ac30fb0 Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Thu, 17 Sep 2026 21:11:45 -0700 Subject: [PATCH 170/660] samples/bpf: Remove unused hash_func01.h header samples/bpf/hash_func01.h has been unused since commit 91dda69b08de ("samples/bpf: Remove the xdp_redirect* utilities"). Remove it. Signed-off-by: Ethan Nelson-Moore Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260918041148.602323-1-enelsonmoore@gmail.com --- samples/bpf/hash_func01.h | 55 --------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 samples/bpf/hash_func01.h diff --git a/samples/bpf/hash_func01.h b/samples/bpf/hash_func01.h deleted file mode 100644 index 38255812e376..000000000000 --- a/samples/bpf/hash_func01.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 - * - * Based on Paul Hsieh's (LGPG 2.1) hash function - * From: http://www.azillionmonkeys.com/qed/hash.html - */ - -#define get16bits(d) (*((const __u16 *) (d))) - -static __always_inline -__u32 SuperFastHash (const char *data, int len, __u32 initval) { - __u32 hash = initval; - __u32 tmp; - int rem; - - if (len <= 0 || data == NULL) return 0; - - rem = len & 3; - len >>= 2; - - /* Main loop */ -#pragma clang loop unroll(full) - for (;len > 0; len--) { - hash += get16bits (data); - tmp = (get16bits (data+2) << 11) ^ hash; - hash = (hash << 16) ^ tmp; - data += 2*sizeof (__u16); - hash += hash >> 11; - } - - /* Handle end cases */ - switch (rem) { - case 3: hash += get16bits (data); - hash ^= hash << 16; - hash ^= ((signed char)data[sizeof (__u16)]) << 18; - hash += hash >> 11; - break; - case 2: hash += get16bits (data); - hash ^= hash << 11; - hash += hash >> 17; - break; - case 1: hash += (signed char)*data; - hash ^= hash << 10; - hash += hash >> 1; - } - - /* Force "avalanching" of final 127 bits */ - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - - return hash; -} From c42da1d3ba9d5067cf3f0676b98bcb3f5f9e3ce1 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 15 Sep 2026 17:07:33 +0200 Subject: [PATCH 171/660] bpf, lsm: Reject writes into the BPF LSM program context The verifier types a load from a tracing-style context by its offset alone: btf_ctx_access() looks the offset up in the attach hook's prototype, and check_mem_access() stamps the resulting type onto the destination register. Nothing tracks what the slot actually holds. That is sound only as long as the context is read-only, and for BPF LSM it is not. lsm_verifier_ops wires ->is_valid_access() straight to btf_ctx_access(), which never looks at enum bpf_access_type, so a store into the context passes verification. Every other user of the helper either goes through bpf_tracing_btf_ctx_access() or open-codes the same type != BPF_READ test first; BPF LSM is the only one that does neither. A program can therefore store a scalar of its choosing into a context slot, read the slot back, and get a register the verifier believes is a trusted pointer of the hook's argument type. Handing that to a kfunc or helper which dereferences a trusted argument turns it into an arbitrary kernel access. Route BPF LSM through bpf_tracing_btf_ctx_access() like everyone else. Fixes: fc611f47f218 ("bpf: Introduce BPF_PROG_TYPE_LSM") Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-3-daniel@iogearbox.net --- kernel/bpf/bpf_lsm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 82c5988417a0..2660a89fc8d7 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -414,8 +414,8 @@ const struct bpf_prog_ops lsm_prog_ops = { }; const struct bpf_verifier_ops lsm_verifier_ops = { - .get_func_proto = bpf_lsm_func_proto, - .is_valid_access = btf_ctx_access, + .get_func_proto = bpf_lsm_func_proto, + .is_valid_access = bpf_tracing_btf_ctx_access, }; /* hooks return 0 or 1 */ From 806fc431e0a399aa50451e19a04327783df7cbc7 Mon Sep 17 00:00:00 2001 From: David Windsor Date: Tue, 15 Sep 2026 17:07:34 +0200 Subject: [PATCH 172/660] bpf: Support passing context output arguments to kfuncs Allows programs to pass a context argument that points to a scalar output value on to a kfunc that writes the result on the program's behalf. A ctx_arg_info entry can now describe a fixed-size PTR_TO_MEM context argument through a new mem_size field, honored in btf_ctx_access(). Loading the argument yields a PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED register of known size, readable but not writable by the program. A kfunc argument tagged with the new "__ctx_out" suffix accepts only that register type with a size matching the pointed-to type, so the only value a program can pass is an output argument from its own context. The pointer stays read-only to the program because the value is trusted by whoever invoked the BPF program. In the first use case, the inode_init_security LSM hook, every LSM receives a shared xattr array and an int *xattr_count that lsm_get_xattr_slot() increments; a program that could store a garbage count would push another LSM's write out of bounds, so only the kfunc itself writes through it. Suggested-by: Kumar Kartikeya Dwivedi Co-developed-by: Daniel Borkmann Signed-off-by: David Windsor Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-4-daniel@iogearbox.net --- Documentation/bpf/kfuncs.rst | 23 +++++++++++++++++++++++ include/linux/bpf.h | 3 +++ kernel/bpf/btf.c | 7 +++++-- kernel/bpf/diagnostics.c | 2 ++ kernel/bpf/verifier.c | 36 +++++++++++++++++++++++++++++++++++- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 6c2c048dccef..3f300118a623 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -315,6 +315,29 @@ However, there is no obligation to prove to the verifier that such a pointer is non-NULL before use, in-line with existing semantics of arena pointers used in a program (or obtained from any other source). +2.3.9 __ctx_out Annotation +-------------------------- + +This annotation is used to indicate that the argument is an output +parameter of the attached hook, passed through from the program's +context. The verifier requires the register to be a trusted read-only +pointer to fixed-size memory, which can only be produced by loading an +argument described by the program's ctx_arg_info from the context. The +program itself cannot write through the pointer; the kfunc may. + +An example is given below:: + + __bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + ...) + { + ... + } + +In this case, a program attached to the ``inode_init_security`` LSM hook +can pass the hook's own ``xattr_count`` argument through to the kfunc, +which claims xattr slots by writing through it. + .. _BPF_kfunc_nodef: 2.4 Using an existing kernel function diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 3abde9a2a375..5033b934ffd9 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -924,6 +924,7 @@ enum bpf_arg_type { ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, @@ -1138,6 +1139,7 @@ struct bpf_insn_access_aux { u32 ref_id; }; }; + u32 mem_size; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; @@ -1716,6 +1718,7 @@ struct bpf_ctx_arg_aux { struct btf *btf; u32 btf_id; u32 ref_id; + u32 mem_size; bool refcounted; }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index e3fb45dcee39..314ecb0e593b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6995,8 +6995,9 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } /* - * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or - * PTR_TO_ARENA (both nullable and non-nullable cases). + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL, + * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size + * PTR_TO_MEM. */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; @@ -7006,8 +7007,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, flag = type_flag(ctx_arg_info->reg_type); if (ctx_arg_info->offset == off && (type == PTR_TO_ARENA || + type == PTR_TO_MEM || (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; + info->mem_size = ctx_arg_info->mem_size; return true; } } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index a2cac59c6639..787932f92cdf 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -984,6 +984,8 @@ const char *bpf_diag_arg_type_plain(enum bpf_arg_type type) return "the address of a stack iterator object for iterator new, next, and destroy calls"; case ARG_PTR_TO_IRQ_FLAG: return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()"; + case ARG_PTR_TO_CTX_OUT: + return "the attach hook's own output argument, loaded directly from the program context"; default: return "a value with one of the accepted pointer or scalar types for this call"; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3539a768b921..18aad4886f9c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6584,6 +6584,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; regs[value_regno].id = info.ref_id; + } else if (base_type(info.reg_type) == PTR_TO_MEM) { + regs[value_regno].mem_size = info.mem_size; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; @@ -8363,6 +8365,9 @@ static const struct bpf_reg_types arena_types = { SCALAR_VALUE, } }; +static const struct bpf_reg_types ctx_out_types = { + .types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED }, +}; static const struct bpf_reg_types alloc_obj_drop_types = { .types = { @@ -8434,6 +8439,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_TASK_WORK] = &map_value_types, [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, [ARG_PTR_TO_ARENA] = &arena_types, + [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, }; static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, @@ -9426,6 +9432,13 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err < 0) return err; break; + case ARG_PTR_TO_CTX_OUT: + if (reg->mem_size != arg_size) { + verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n", + reg_arg_name(env, argno), arg_size, reg->mem_size); + return -EINVAL; + } + break; case ARG_PTR_TO_RES_SPIN_LOCK: { int flags; @@ -12142,6 +12155,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__ctx_out"); +} + static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__arena__nullable") || @@ -12905,7 +12923,23 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) arg_type = ARG_PTR_TO_RES_SPIN_LOCK; - else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) { + if (!btf_type_is_scalar(ref_t)) { + verbose(env, "%s __ctx_out argument must point to a scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE; + } else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) arg_type = ARG_PTR_TO_FUNC; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { if (!bpf_jit_supports_arena_args()) { From ff0d1c0915d1fce9b4b8613a8b52b0da51463d87 Mon Sep 17 00:00:00 2001 From: David Windsor Date: Tue, 15 Sep 2026 17:07:35 +0200 Subject: [PATCH 173/660] bpf, lsm: Let BPF LSM provide xattrs at inode creation Many in-kernel LSMs (SELinux, Smack, IMA) store security labels in extended attributes. For these LSMs, atomic labeling during inode creation is critical: if the inode becomes accessible before its xattr is set, it is briefly unlabeled, which can disrupt LSMs making policy decisions based on file labels. Existing LSMs solve this by setting xattrs in the inode_init_security hook, which runs before the inode becomes accessible. BPF LSM programs currently lack this capability because the hook uses an output parameter (xattr_count) that BPF programs cannot write to, and existing kfuncs like bpf_set_dentry_xattr() require a dentry that isn't available until after the inode is accessible. Add a bpf_inode_init_xattr() kfunc that takes the hook's own xattrs and xattr_count arguments, passed through from the program's context, and claims a slot via lsm_get_xattr_slot() on the program's behalf. The xattr_count output argument is exposed to inode_init_security programs as trusted read-only memory, so programs can pass it to the kfunc but cannot modify the count themselves. Reserve BPF_LSM_INODE_INIT_XATTRS slots in bpf_lsm_blob_sizes the way every other xattr-providing LSM does, for the life of the kernel. The framework keys the collection off the reserved slot count, so a kernel built with CONFIG_BPF_LSM=y now allocates the xattr array on every inode creation, whether or not a program sits on the hook. Give the hook a strong prototype in bpf_lsm_proto.c so that its qstr and xattrs arguments are marked __nullable. Both can be NULL for some callers. Without the annotation the verifier otherwise hands the program a trusted non-NULL pointer which it dereferences. Also, keep the hook out of the sleepable set. inode_init_security runs inside the transaction creating the inode, with a journal handle held on ext4 and btrfs and the parent's i_rwsem down, which is why everything on the path allocates GFP_NOFS. Co-developed-by: Daniel Borkmann Signed-off-by: David Windsor Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-5-daniel@iogearbox.net --- fs/bpf_fs_kfuncs.c | 99 ++++++++++++++++++++++++++++++++++++++ include/linux/bpf_lsm.h | 11 ++++- kernel/bpf/bpf_lsm.c | 22 ++++++++- kernel/bpf/bpf_lsm_proto.c | 15 ++++++ security/bpf/hooks.c | 1 + 5 files changed, 145 insertions(+), 3 deletions(-) diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index 6cb877267978..c51c3ae8063b 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -328,6 +329,89 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_ return ret; } +static int bpf_inode_init_xattrs_claimed(const struct xattr *xattrs, int xattr_count) +{ + const size_t suffix_len = sizeof(XATTR_BPF_LSM_SUFFIX) - 1; + int i, claimed = 0; + + for (i = 0; i < xattr_count; i++) { + const char *name = xattrs[i].name; + + if (name && !strncmp(name, XATTR_BPF_LSM_SUFFIX, suffix_len)) + claimed++; + } + return claimed; +} + +/** + * bpf_inode_init_xattr - attach a xattr to an inode that is being created + * @xattrs: xattr array the inode_init_security hook was handed + * @xattr_count__ctx_out: xattr count the inode_init_security hook was handed + * @name__str: name of the xattr + * @value_p: xattr value + * + * Claim one of the slots the BPF LSM reserved in the xattr array, so that + * the xattr is written out as part of the transaction creating the inode. + * + * For security reasons, only *name__str* with prefix "security.bpf." is + * allowed. The slot stores the name without that "security." prefix, which + * the filesystem's initxattrs() callback puts back. + * + * At most BPF_LSM_INODE_INIT_XATTRS xattrs can be attached to one inode, + * matching the number of slots the BPF LSM reserves. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + const char *name__str, + const struct bpf_dynptr *value_p) +{ + const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + int *xattr_count = xattr_count__ctx_out; + const char *suffix; + struct xattr *slot; + const void *value; + size_t name_len; + u32 value_len; + char *buf; + + if (!match_security_bpf_prefix(name__str)) + return -EPERM; + if (bpf_inode_init_xattrs_claimed(xattrs, *xattr_count) >= + BPF_LSM_INODE_INIT_XATTRS) + return -ENOSPC; + + suffix = name__str + XATTR_SECURITY_PREFIX_LEN; + name_len = strlen(suffix); + if (name_len <= sizeof(XATTR_BPF_LSM_SUFFIX) - 1 || + name_len > XATTR_NAME_MAX - XATTR_SECURITY_PREFIX_LEN) + return -EINVAL; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data(value_ptr, value_len); + if (!value) + return -EINVAL; + if (value_len > XATTR_SIZE_MAX) + return -E2BIG; + + buf = kmalloc(value_len + name_len + 1, GFP_NOWAIT | __GFP_NOWARN); + if (!buf) + return -ENOMEM; + memcpy(buf, value, value_len); + memcpy(buf + value_len, suffix, name_len + 1); + + slot = lsm_get_xattr_slot(xattrs, xattr_count); + if (!slot) { + kfree(buf); + return -ENOSPC; + } + slot->value = buf; + slot->value_len = value_len; + slot->name = buf + value_len; + return 0; +} + #ifdef CONFIG_CGROUPS /** * bpf_cgroup_read_xattr - read xattr of a cgroup's node in cgroupfs @@ -425,6 +509,7 @@ BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_inode_init_xattr) #ifdef CONFIG_NET BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU) #endif @@ -436,10 +521,24 @@ BTF_ID(func, bpf_set_dentry_xattr) BTF_ID(func, bpf_remove_dentry_xattr) BTF_SET_END(bpf_fs_kfunc_lsm_only_ids) +BTF_ID_LIST_SINGLE(bpf_inode_init_xattr_ids, func, bpf_inode_init_xattr) + +BTF_SET_START(bpf_inode_init_xattr_hooks) +BTF_ID(func, bpf_lsm_inode_init_security) +BTF_SET_END(bpf_inode_init_xattr_hooks) + static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) { if (!btf_id_set8_contains(&bpf_fs_kfunc_set_ids, kfunc_id)) return 0; + if (kfunc_id == bpf_inode_init_xattr_ids[0]) { + if (prog->type != BPF_PROG_TYPE_LSM || + prog->expected_attach_type != BPF_LSM_MAC || + !btf_id_set_contains(&bpf_inode_init_xattr_hooks, + prog->aux->attach_btf_id)) + return -EACCES; + return 0; + } if (prog->type == BPF_PROG_TYPE_LSM) return 0; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index dda272d78f01..eb4a38eef87e 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init; #include #undef LSM_HOOK +/* + * Number of xattr slots the BPF LSM reserves in the array handed to + * security_inode_init_security(), i.e. the maximum number of labels + * a policy may attach to an inode while it is being created. + */ +#define BPF_LSM_INODE_INIT_XATTRS 2 + struct bpf_storage_blob { struct bpf_local_storage __rcu *storage; }; @@ -28,7 +35,7 @@ struct bpf_storage_blob { extern struct lsm_blob_sizes bpf_lsm_blob_sizes; int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog); + struct bpf_prog *prog); bool bpf_lsm_is_sleepable_hook(u32 btf_id); bool bpf_lsm_is_trusted(const struct bpf_prog *prog); @@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog) } static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { return -EOPNOTSUPP; } diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 2660a89fc8d7..f58dce888ff4 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -116,8 +116,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, } #endif +BTF_ID_LIST_SINGLE(bpf_lsm_inode_init_security_btf_id, func, + bpf_lsm_inode_init_security) + int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { u32 btf_id = prog->aux->attach_btf_id; const char *func_name = prog->aux->attach_func_name; @@ -140,6 +143,23 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, return -EINVAL; } + if (btf_id == bpf_lsm_inode_init_security_btf_id[0]) { + /* + * inode, dir, qstr, xattrs, xattr_count + * + * The trusted pointer this hands the program is only as + * trustworthy as the context it is loaded from, which + * lsm_verifier_ops keeps read-only. + */ + static const struct bpf_ctx_arg_aux xattr_count_arg_info = { + .offset = 4 * sizeof(u64), + .reg_type = PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED, + .mem_size = sizeof(int), + }; + + return bpf_prog_ctx_arg_info_init(prog, &xattr_count_arg_info, 1); + } + return 0; } diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c index 44a54fd8045e..4a776df76cbb 100644 --- a/kernel/bpf/bpf_lsm_proto.c +++ b/kernel/bpf/bpf_lsm_proto.c @@ -4,6 +4,7 @@ */ #include +#include #include /* @@ -17,3 +18,17 @@ int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, { return 0; } + +/* + * Strong definition of the inode_init_security() BPF LSM hook. Both the + * qstr and the xattr array are NULL for some callers, so the __nullable + * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before + * dereferencing them or handing them to bpf_inode_init_xattr(). + */ +int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr__nullable, + struct xattr *xattrs__nullable, + int *xattr_count) +{ + return -EOPNOTSUPP; +} diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 7b98f5d1e2be..8f8c3de3035f 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c @@ -33,6 +33,7 @@ static int __init bpf_lsm_init(void) struct lsm_blob_sizes bpf_lsm_blob_sizes __ro_after_init = { .lbs_inode = sizeof(struct bpf_storage_blob), + .lbs_xattr_count = BPF_LSM_INODE_INIT_XATTRS, }; DEFINE_LSM(bpf) = { From 3479e396eef4db6514ab59d2ec889ae237869486 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 15 Sep 2026 17:07:36 +0200 Subject: [PATCH 174/660] bpf, lsm: Mark the BPF LSM hook overrides noinline The stubs that bpf_lsm_proto.c overrides are generated as: #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ __weak noinline RET bpf_lsm_##NAME(__VA_ARGS__) \ { \ return DEFAULT; \ } Mark the overrides noinline as well so they hold up the same contract as the stubs they replace. Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-6-daniel@iogearbox.net --- kernel/bpf/bpf_lsm_proto.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c index 4a776df76cbb..b81179ae2e08 100644 --- a/kernel/bpf/bpf_lsm_proto.c +++ b/kernel/bpf/bpf_lsm_proto.c @@ -13,8 +13,8 @@ * explicitly enforces that BPF LSM programs check for NULL before attempting to * dereference it. */ -int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, - unsigned long prot, unsigned long flags) +noinline int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, + unsigned long prot, unsigned long flags) { return 0; } @@ -25,10 +25,10 @@ int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before * dereferencing them or handing them to bpf_inode_init_xattr(). */ -int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, - const struct qstr *qstr__nullable, - struct xattr *xattrs__nullable, - int *xattr_count) +noinline int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr__nullable, + struct xattr *xattrs__nullable, + int *xattr_count) { return -EOPNOTSUPP; } From e5d960cfe1d3a1fd083cc258613928857f287168 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 15 Sep 2026 17:07:37 +0200 Subject: [PATCH 175/660] selftests/bpf: Test that the BPF LSM context is read-only Add a new BPF selftest asserting that a store into the context of an LSM program must be rejected. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -a verifier_lsm [...] #648/1 verifier_lsm/lsm bpf prog with -4095~0 retval. test 1:OK #648/2 verifier_lsm/lsm bpf prog with -4095~0 retval. test 2:OK #648/3 verifier_lsm/lsm bpf prog with -4095~0 retval. test 4:OK #648/4 verifier_lsm/lsm bpf prog with -4095~0 retval. test 5:OK #648/5 verifier_lsm/lsm bpf prog with -4095~0 retval. test 6:OK #648/6 verifier_lsm/lsm retval load must reset stale register bounds:OK #648/7 verifier_lsm/lsm ctx is read-only:OK #648/8 verifier_lsm/lsm bpf prog with bool retval. test 1:OK #648/9 verifier_lsm/lsm bpf prog with bool retval. test 2:OK #648/10 verifier_lsm/lsm bpf prog with bool retval. test 3:OK #648/11 verifier_lsm/lsm bpf prog with bool retval. test 4:OK #648/12 verifier_lsm/lsm bpf prog with void retval. test 1:OK #648/13 verifier_lsm/lsm bpf prog with void retval. test 2:OK #648/14 verifier_lsm/lsm disabled hook: getprocattr:OK #648/15 verifier_lsm/lsm disabled hook: setprocattr:OK #648/16 verifier_lsm/lsm disabled hook: ismaclabel:OK #648/17 verifier_lsm/not null checking nullable pointer in bpf_lsm_mmap_file:OK #648/18 verifier_lsm/null checking nullable pointer in bpf_lsm_mmap_file:OK #648/19 verifier_lsm/sleepable lsm_cgroup program is rejected:OK #648 verifier_lsm:OK Summary: 1/19 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-7-daniel@iogearbox.net --- tools/testing/selftests/bpf/progs/verifier_lsm.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index c724bf389f5c..8fbf7862265f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c @@ -212,4 +212,17 @@ __naked int retval_load_resets_bounds(void *ctx) ::: __clobber_all); } +SEC("lsm/file_mprotect") +__description("lsm ctx is read-only") +__failure __msg("invalid bpf_context access") +__naked int reject_ctx_write(void *ctx) +{ + asm volatile ( + "r6 = 0;" + "*(u64 *)(r1 + 0) = r6;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 2757e9e53900a62b4412b0a707f738587110f975 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 15 Sep 2026 17:07:38 +0200 Subject: [PATCH 176/660] selftests/bpf: Add verifier tests for the __ctx_out plumbing Add BPF selftests to cover what a BPF program may and may not hand to bpf_inode_init_xattr() as the inode_init_security hook's args. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t verifier_lsm_init_xattr [...] #649/1 verifier_lsm_init_xattr/reject_count_write:OK #649/2 verifier_lsm_init_xattr/allow_count_read:OK #649/3 verifier_lsm_init_xattr/reject_unchecked_xattrs:OK #649/4 verifier_lsm_init_xattr/reject_unchecked_qstr:OK #649/5 verifier_lsm_init_xattr/allow_spilled_count:OK #649/6 verifier_lsm_init_xattr/reject_forged_count:OK #649/7 verifier_lsm_init_xattr/reject_stack_count:OK #649/8 verifier_lsm_init_xattr/reject_other_ctx_arg:OK #649/9 verifier_lsm_init_xattr/reject_null_count:OK #649/10 verifier_lsm_init_xattr/reject_shifted_count:OK #649/11 verifier_lsm_init_xattr/reject_var_shifted_count:OK #649/12 verifier_lsm_init_xattr/reject_ctx_forged_count:OK #649/13 verifier_lsm_init_xattr/allow_count_via_subprog:OK #649/14 verifier_lsm_init_xattr/reject_shifted_xattrs:OK #649/15 verifier_lsm_init_xattr/reject_sleepable:OK #649/16 verifier_lsm_init_xattr/reject_lsm_cgroup:OK #649/17 verifier_lsm_init_xattr/reject_wrong_hook:OK #649 verifier_lsm_init_xattr:OK Summary: 1/17 PASSED, 0 SKIPPED, 0/0 FAILED Co-developed-by: David Windsor Signed-off-by: David Windsor Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-8-daniel@iogearbox.net --- tools/testing/selftests/bpf/bpf_kfuncs.h | 19 +- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_lsm_init_xattr.c | 245 ++++++++++++++++++ 3 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c diff --git a/tools/testing/selftests/bpf/bpf_kfuncs.h b/tools/testing/selftests/bpf/bpf_kfuncs.h index ae71e9b69051..1b408fe9e0fb 100644 --- a/tools/testing/selftests/bpf/bpf_kfuncs.h +++ b/tools/testing/selftests/bpf/bpf_kfuncs.h @@ -78,18 +78,29 @@ extern void bpf_key_put(struct bpf_key *key) __ksym; extern int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_ptr, const struct bpf_dynptr *sig_ptr, struct bpf_key *trusted_keyring) __ksym; - -struct dentry; -/* Description +/* + * Description * Returns xattr of a dentry * Returns * Error code */ +struct dentry; extern int bpf_get_dentry_xattr(struct dentry *dentry, const char *name, struct bpf_dynptr *value_ptr) __ksym __weak; - extern int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags) __ksym __weak; extern int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) __ksym __weak; +/* + * Description + * Attach a xattr to an inode that is being created, from a program on the + * inode_init_security LSM hook. *xattrs* and *xattr_count* must be the + * hook's own arguments, passed through unmodified. + * Returns + * 0 on success, a negative value on error + */ +struct xattr; +extern int bpf_inode_init_xattr(struct xattr *xattrs, int *xattr_count, + const char *name__str, + const struct bpf_dynptr *value_p) __ksym __weak; #endif diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 7732df9bc870..973bbeda9318 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -130,6 +130,7 @@ #include "verifier_bits_iter.skel.h" #include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" +#include "verifier_lsm_init_xattr.skel.h" #include "verifier_jit_inline.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" @@ -294,6 +295,7 @@ void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_pack void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } void test_verifier_set_retval(void) { RUN(verifier_set_retval); } void test_verifier_lsm(void) { RUN(verifier_lsm); } +void test_verifier_lsm_init_xattr(void) { RUN(verifier_lsm_init_xattr); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c new file mode 100644 index 000000000000..b706bf17b556 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include +#include +#include "bpf_kfuncs.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_zone[] = "security.bpf.zone"; +char value_buf[8] = "z"; +int scratch_count; + +SEC("lsm/inode_init_security") +__failure __msg("cannot write into rdonly_trusted_mem") +int BPF_PROG(reject_count_write, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + *xattr_count = 0; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_read, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = *xattr_count; + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_unchecked_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid mem access 'trusted_ptr_or_null_'") +int BPF_PROG(reject_unchecked_qstr, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = qstr->len; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_spilled_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int *saved = xattr_count; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, saved, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_forged_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &scratch_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_stack_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int local = 0; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &local, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_other_ctx_arg, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, (int *)xattrs, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_null_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, NULL, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("dereference of modified rdonly_trusted_mem ptr") +int BPF_PROG(reject_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + 1, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("variable rdonly_trusted_mem access var_off=") +int BPF_PROG(reject_var_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + (scratch_count & 1), + xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid bpf_context access") +int reject_ctx_forged_count(unsigned long long *ctx) +{ + volatile unsigned long long *slot = ctx; + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + slot[4] = 0; + bpf_inode_init_xattr((struct xattr *)(long)slot[3], + (int *)(long)slot[4], xattr_zone, &value); + return 0; +} + +static __noinline int claim_via_subprog(struct xattr *xattrs, int *xattr_count, + struct bpf_dynptr *value) +{ + return bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, value); +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_via_subprog, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + claim_via_subprog(xattrs, xattr_count, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("access beyond struct xattr at off 24") +int BPF_PROG(reject_shifted_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs + 1, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm.s/inode_init_security") +__failure __msg("bpf_lsm_inode_init_security is not sleepable") +int BPF_PROG(reject_sleepable, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + return 0; +} + +SEC("lsm_cgroup/inode_init_security") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_lsm_cgroup, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_setxattr") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_wrong_hook, struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, const void *value, size_t size, int flags) +{ + struct bpf_dynptr val; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &val); + bpf_inode_init_xattr(NULL, &scratch_count, xattr_zone, &val); + return 0; +} From 90dd01b6c1f338c19584b86d7ecb0dc70f497439 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 15 Sep 2026 17:07:39 +0200 Subject: [PATCH 177/660] selftests/bpf: Add tests for BPF LSM inode init labelling Exercise bpf_inode_init_xattr() in combination with a policy example via BPF LSM. A program on the inode_init_security hook labels new files and directories, inherits a zone label from the parent directory, and has claims refused for names outside the security.bpf. prefix and for a name that would exceed XATTR_NAME_MAX once the prefix is put back as well as other corner case tests that should get rejected. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t lsm_inode_init_xattr [...] #226/1 lsm_inode_init_xattr/init_labels:OK #226/2 lsm_inode_init_xattr/inherit_from_parent:OK #226/3 lsm_inode_init_xattr/refused_claims:OK #226/4 lsm_inode_init_xattr/null_xattrs:OK #226/5 lsm_inode_init_xattr/shared_budget:OK #226/6 lsm_inode_init_xattr/value_shapes:OK #226 lsm_inode_init_xattr:OK Summary: 1/6 PASSED, 0 SKIPPED, 0/0 FAILED Co-developed-by: David Windsor Signed-off-by: David Windsor Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260915150739.284189-9-daniel@iogearbox.net --- tools/testing/selftests/bpf/config | 2 + .../bpf/prog_tests/lsm_inode_init_xattr.c | 399 ++++++++++++++++++ .../bpf/progs/lsm_inode_init_xattr.c | 98 +++++ .../bpf/progs/lsm_inode_init_xattr_budget.c | 37 ++ .../bpf/progs/lsm_inode_init_xattr_value.c | 47 +++ 5 files changed, 583 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index 2f79688dcf7c..d292cb60a5a4 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -124,6 +124,8 @@ CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_SYN_COOKIES=y CONFIG_TEST_BPF=m +CONFIG_TMPFS=y +CONFIG_TMPFS_XATTR=y CONFIG_UDMABUF=y CONFIG_USERFAULTFD=y CONFIG_VSOCKETS=y diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c new file mode 100644 index 000000000000..b91c5c659542 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lsm_inode_init_xattr.skel.h" +#include "lsm_inode_init_xattr_budget.skel.h" +#include "lsm_inode_init_xattr_value.skel.h" + +#define INIT_XATTRS_MAX 2 +#define VALUE_SIZE_MAX (64 * 1024) +#define TESTDIR "/tmp/test_progs_lsm_init_xattr" +#define RAMFSDIR "/tmp/test_progs_lsm_init_xattr_ramfs" + +static bool testdir_mounted; +static bool ramfsdir_mounted; + +static int testdir_setup(void) +{ + if (mkdir(TESTDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("tmpfs", TESTDIR, "tmpfs", 0, NULL)) + return -errno; + testdir_mounted = true; + if (mkdir(RAMFSDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("ramfs", RAMFSDIR, "ramfs", 0, NULL)) + return -errno; + ramfsdir_mounted = true; + return 0; +} + +static void testdir_cleanup(void) +{ + if (ramfsdir_mounted) + umount(RAMFSDIR); + rmdir(RAMFSDIR); + if (testdir_mounted) + umount(TESTDIR); + rmdir(TESTDIR); +} + +static bool lsm_is_active(const char *name) +{ + char buf[512], *tok; + int fd, len; + + fd = open("/sys/kernel/security/lsm", O_RDONLY); + if (fd < 0) + return true; + len = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (len <= 0) + return true; + buf[len] = '\0'; + for (tok = strtok(buf, ",\n"); tok; tok = strtok(NULL, ",\n")) + if (!strcmp(tok, name)) + return true; + return false; +} + +static int read_label(const char *path, const char *name, char *buf, size_t sz) +{ + int ret = getxattr(path, name, buf, sz); + + return ret < 0 ? -errno : ret; +} + +static void assert_label(const char *path, const char *name, const char *want) +{ + char buf[64] = {}; + int ret; + + ret = read_label(path, name, buf, sizeof(buf)); + if (!ASSERT_EQ(ret, (int)strlen(want) + 1, name)) + return; + ASSERT_STREQ(buf, want, name); +} + +static struct lsm_inode_init_xattr *policy_attach(void) +{ + struct lsm_inode_init_xattr *skel; + + skel = lsm_inode_init_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return NULL; + + skel->bss->monitored_pid = getpid(); + if (!ASSERT_OK(lsm_inode_init_xattr__attach(skel), "skel_attach")) { + lsm_inode_init_xattr__destroy(skel); + return NULL; + } + return skel; +} + +static void test_init_labels(void) +{ + struct lsm_inode_init_xattr *skel; + const char *file = TESTDIR "/file"; + const char *subdir = TESTDIR "/subdir"; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_OK(skel->data->zone_err, "zone_err"); + ASSERT_OK(skel->data->origin_err, "origin_err"); + assert_label(file, "security.bpf.zone", "default"); + assert_label(file, "security.bpf.origin", "created"); + + if (!ASSERT_OK(mkdir(subdir, 0755), "mkdir")) + goto out; + assert_label(subdir, "security.bpf.zone", "default"); + assert_label(subdir, "security.bpf.origin", "created"); + rmdir(subdir); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_inherit_from_parent(void) +{ + const char *zonedir = TESTDIR "/zoned"; + const char *file = TESTDIR "/zoned/file"; + struct lsm_inode_init_xattr *skel; + int dirfd = -1, fd = -1, err; + struct { + char v[32]; + __u32 len; + } label = {}; + + if (!ASSERT_OK(mkdir(zonedir, 0755), "mkdir_zoned")) + return; + + skel = policy_attach(); + if (!skel) + goto out_dir; + + dirfd = open(zonedir, O_RDONLY | O_DIRECTORY); + if (!ASSERT_GE(dirfd, 0, "open_zoned")) + goto out; + + strncpy(label.v, "restricted", sizeof(label.v) - 1); + label.len = strlen("restricted") + 1; + + err = bpf_map_update_elem(bpf_map__fd(skel->maps.inode_zone), &dirfd, + &label, BPF_ANY); + if (!ASSERT_OK(err, "seed_parent_zone")) + goto out; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->bss->inherited, 1, "inherited"); + ASSERT_OK(skel->data->zone_err, "zone_err"); + assert_label(file, "security.bpf.zone", "restricted"); + assert_label(file, "security.bpf.origin", "created"); +out: + if (fd >= 0) + close(fd); + if (dirfd >= 0) + close(dirfd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +out_dir: + rmdir(zonedir); +} + +static void test_refused_claims(void) +{ + const char *file = TESTDIR "/refused"; + struct lsm_inode_init_xattr *skel; + char buf[64]; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->data->overflow_err, -ENOSPC, "overflow_err"); + ASSERT_EQ(read_label(file, "security.bpf.overflow", buf, sizeof(buf)), + -ENODATA, "overflow_absent"); + + ASSERT_EQ(skel->data->toolong_err, -EINVAL, "toolong_err"); + + ASSERT_EQ(skel->data->selinux_err, -EPERM, "selinux_err"); + ASSERT_EQ(skel->data->user_err, -EPERM, "user_err"); + + if (!lsm_is_active("selinux")) + ASSERT_EQ(read_label(file, "security.selinux", buf, sizeof(buf)), + -ENODATA, "selinux_absent"); + ASSERT_EQ(read_label(file, "user.zone", buf, sizeof(buf)), + -ENODATA, "user_absent"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_null_xattrs(void) +{ + const char *file = RAMFSDIR "/file"; + struct lsm_inode_init_xattr *skel; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + ASSERT_EQ(skel->data->zone_err, -EOPNOTSUPP, "zone_err"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_shared_budget(void) +{ + struct lsm_inode_init_xattr_budget *skel[INIT_XATTRS_MAX + 1] = {}; + struct bpf_link *link[INIT_XATTRS_MAX + 1] = {}; + const char *file = TESTDIR "/budget"; + int claimed = 0, refused = 0; + int i, fd = -1; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + skel[i] = lsm_inode_init_xattr_budget__open(); + if (!ASSERT_OK_PTR(skel[i], "skel_open")) + goto out; + + snprintf(skel[i]->rodata->xattr_name, + sizeof(skel[i]->rodata->xattr_name), + "security.bpf.slot%d", i); + + if (!ASSERT_OK(lsm_inode_init_xattr_budget__load(skel[i]), + "skel_load")) + goto out; + + skel[i]->bss->monitored_pid = getpid(); + + link[i] = bpf_program__attach_lsm(skel[i]->progs.claim_one); + if (!ASSERT_OK_PTR(link[i], "attach")) + goto out; + } + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + int err = skel[i]->data->claim_err; + + ASSERT_TRUE(skel[i]->bss->hook_ran, "hook_ran"); + if (!err) + claimed++; + else if (ASSERT_EQ(err, -ENOSPC, "claim_err")) + refused++; + } + + ASSERT_EQ(claimed, INIT_XATTRS_MAX, "claimed"); + ASSERT_EQ(refused, 1, "refused"); +out: + if (fd >= 0) + close(fd); + remove(file); + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + bpf_link__destroy(link[i]); + lsm_inode_init_xattr_budget__destroy(skel[i]); + } +} + +static int claim_value(const char *file, __u32 len, __u64 flags, + int hook_retval, int *open_errno) +{ + struct lsm_inode_init_xattr_value *skel; + struct bpf_link *link = NULL; + int err = 1, fd; + + skel = lsm_inode_init_xattr_value__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return 1; + + skel->rodata->value_len = len; + skel->rodata->dynptr_flags = flags; + skel->rodata->hook_retval = hook_retval; + + if (!ASSERT_OK(lsm_inode_init_xattr_value__load(skel), "skel_load")) + goto out; + + skel->bss->monitored_pid = getpid(); + + link = bpf_program__attach_lsm(skel->progs.claim_value); + if (!ASSERT_OK_PTR(link, "attach")) + goto out; + + fd = open(file, O_CREAT | O_RDWR | O_EXCL, 0644); + *open_errno = fd < 0 ? errno : 0; + if (fd >= 0) + close(fd); + + if (ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + err = skel->data->claim_err; +out: + bpf_link__destroy(link); + lsm_inode_init_xattr_value__destroy(skel); + return err; +} + +static void test_value_shapes(void) +{ + const char *file = TESTDIR "/value"; + char buf[64]; + int err, open_errno; + + remove(file); + + err = claim_value(file, 0, 0, 0, &open_errno); + ASSERT_OK(err, "empty_value_err"); + ASSERT_OK(open_errno, "empty_value_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + 0, "empty_value_len"); + remove(file); + + err = claim_value(file, sizeof(buf), 1, 0, &open_errno); + ASSERT_EQ(err, -EINVAL, "bad_dynptr_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "bad_dynptr_absent"); + remove(file); + + err = claim_value(file, VALUE_SIZE_MAX + 1, 0, 0, &open_errno); + ASSERT_EQ(err, -E2BIG, "oversized_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "oversized_absent"); + remove(file); + + err = claim_value(file, 8, 0, -EPERM, &open_errno); + ASSERT_OK(err, "denied_claim_err"); + ASSERT_EQ(open_errno, EPERM, "denied_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENOENT, "denied_absent"); + remove(file); +} + +void test_lsm_inode_init_xattr(void) +{ + if (!ASSERT_OK(testdir_setup(), "testdir_setup")) + goto out; + + if (test__start_subtest("init_labels")) + test_init_labels(); + if (test__start_subtest("inherit_from_parent")) + test_inherit_from_parent(); + if (test__start_subtest("refused_claims")) + test_refused_claims(); + if (test__start_subtest("null_xattrs")) + test_null_xattrs(); + if (test__start_subtest("shared_budget")) + test_shared_budget(); + if (test__start_subtest("value_shapes")) + test_value_shapes(); +out: + testdir_cleanup(); +} diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c new file mode 100644 index 000000000000..77b37cf6165a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define LABEL_MAX 32 + +struct label { + char v[LABEL_MAX]; + __u32 len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_INODE_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct label); +} inode_zone SEC(".maps"); + +__u32 monitored_pid; + +const char xattr_zone[] = "security.bpf.zone"; +const char xattr_origin[] = "security.bpf.origin"; +const char xattr_overflow[] = "security.bpf.overflow"; +/* One byte over XATTR_NAME_MAX once "security." is prepended again. */ +const char xattr_toolong[] = "security.bpf." "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const char xattr_selinux[] = "security.selinux"; +const char xattr_user[] = "user.zone"; + +char zone_scratch[LABEL_MAX]; +char origin_value[] = "created"; + +__u32 hook_ran; +__s32 zone_err = 1; +__s32 origin_err = 1; +__s32 overflow_err = 1; +__s32 toolong_err = 1; +__s32 selinux_err = 1; +__s32 user_err = 1; +__u32 inherited; + +SEC("lsm/inode_init_security") +int BPF_PROG(init_label, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + struct label *parent; + int len = 0; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + /* The filesystem takes no xattrs at inode creation. */ + zone_err = -EOPNOTSUPP; + return 0; + } + __builtin_memset(zone_scratch, 0, LABEL_MAX); + + if (dir) { + parent = bpf_inode_storage_get(&inode_zone, dir, 0, 0); + if (parent && parent->len > 0 && parent->len <= LABEL_MAX) { + len = parent->len; + __builtin_memcpy(zone_scratch, parent->v, LABEL_MAX); + inherited = 1; + } + } + if (!len) { + __builtin_memcpy(zone_scratch, "default", sizeof("default")); + len = sizeof("default"); + } + + bpf_dynptr_from_mem(zone_scratch, len, 0, &value); + /* Refused before a slot is claimed, so the budget below is intact. */ + toolong_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_toolong, + &value); + zone_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, + &value); + + bpf_dynptr_from_mem(origin_value, sizeof(origin_value), 0, &value); + origin_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_origin, + &value); + + overflow_err = bpf_inode_init_xattr(xattrs, xattr_count, + xattr_overflow, &value); + + selinux_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_selinux, + &value); + user_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_user, + &value); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c new file mode 100644 index 000000000000..08dd93a1db11 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_name[32] = "security.bpf.slot"; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +char claim_value[] = "v"; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_one, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(claim_value, sizeof(claim_value), 0, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_name, + &value); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c new file mode 100644 index 000000000000..6a879a758601 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include +#include +#include +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define VALUE_MAX (64 * 1024 + 1) + +const char xattr_value[] = "security.bpf.value"; + +/* Shape of the value handed to the kfunc, and what the hook returns after. */ +const volatile __u32 value_len; +const volatile __u64 dynptr_flags; +const volatile __s32 hook_retval; + +char value_src[VALUE_MAX]; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_value, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + __u32 len = value_len; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + if (len > VALUE_MAX) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(value_src, len, dynptr_flags, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_value, + &value); + return hook_retval; +} From 48ea1ba704aa3a9975e917411ee05bbe67d65744 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Mon, 21 Sep 2026 11:08:54 +0800 Subject: [PATCH 178/660] mm/bpf: Add bpf_proactive_reclaim kfunc BPF programs can observe memory pressure on a cgroup, e.g. refault stats via bpf_mem_cgroup_page_state(), but cannot act on it: triggering reclaim requires writing to memory.reclaim, which BPF cannot do. Add bpf_proactive_reclaim(), a sleepable kfunc performing one proactive reclaim pass on a memcg, like a write to memory.reclaim but without retrying until the target is reached, so that when and how hard to reclaim is BPF policy rather than hard-coded thresholds. The reclaim target of a single call is capped at MEMCG_CHARGE_BATCH, as high_work_func() does for memory.high; reclaiming more is left to the program, which can call the kfunc once per bpf_wq callback and stop at any point. It is limited to BPF_PROG_TYPE_SYSCALL, because other sleepable programs may run with filesystem locks held, on which the reclaim path could deadlock via filesystem shrinkers. Convert MIN_SWAPPINESS, MAX_SWAPPINESS and SWAPPINESS_ANON_ONLY from macros to an enum so that they are emitted into BTF and usable from BPF programs via vmlinux.h. Signed-off-by: Hui Zhu Acked-by: Shakeel Butt Reviewed-by: JP Kobryn Reviewed-by: Barry Song Link: https://lore.kernel.org/r/20260921030854.17858-1-hui.zhu@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- mm/bpf_memcontrol.c | 65 ++++++++++++++++++++++++++++++++++++++++++++- mm/internal.h | 10 ++++--- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49d7647..d8f579c28560 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -8,6 +8,8 @@ #include #include +#include "internal.h" + __bpf_kfunc_start_defs(); /** @@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg) mem_cgroup_flush_stats(memcg); } +/** + * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup + * @memcg: the target memory cgroup to reclaim from. + * @size: the amount of memory to reclaim, in bytes, clamped to + * MEMCG_CHARGE_BATCH. + * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS, + * MAX_SWAPPINESS], or one of the special modes: -1 to use + * the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to + * reclaim only anon folios. + * + * Performs one proactive reclaim pass on @memcg, like a write to + * memory.reclaim but without retrying until @size is reached. Call it + * repeatedly to reclaim more than one batch. + * + * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs + * may run with filesystem locks held, which the reclaim path can deadlock + * on via filesystem shrinkers. + * + * Return: The amount of memory reclaimed, in bytes, or a negative error. + */ +__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg, + unsigned long size, + int swappiness) +{ + unsigned long nr_reclaimed; + unsigned long nr_pages; + + if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) { + if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS) + return -EINVAL; + } + + if (size < PAGE_SIZE) + return -EINVAL; + + nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH); + + nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL, + MEMCG_RECLAIM_MAY_SWAP | + MEMCG_RECLAIM_PROACTIVE, + swappiness == -1 ? NULL : &swappiness); + + return nr_reclaimed * PAGE_SIZE; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) BTF_KFUNCS_END(bpf_memcontrol_kfuncs) +BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs) +BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE) +BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs) + static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { .owner = THIS_MODULE, .set = &bpf_memcontrol_kfuncs, }; +static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_memcontrol_reclaim_kfuncs, +}; + static int __init bpf_memcontrol_init(void) { int err; err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_memcontrol_kfunc_set); - if (err) + if (err) { pr_warn("error while registering bpf memcontrol kfuncs: %d", err); + return err; + } + + err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &bpf_memcontrol_reclaim_kfunc_set); + if (err) + pr_warn("error registering bpf reclaim kfuncs: %d\n", err); return err; } diff --git a/mm/internal.h b/mm/internal.h index 7c326965a94f..9e3be236f18c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -77,11 +77,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 +enum { + MIN_SWAPPINESS = 0, + MAX_SWAPPINESS = 200, -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + /* Just reclaim from anon folios in proactive memory reclaim */ + SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1, +}; unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, From 75ab33a0a50f85085c918c0d2211ca83d18cbb27 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:47 -0700 Subject: [PATCH 179/660] selftests/bpf: Keep headers off the generic link command line The generic '$(OUTPUT)/%:%.c' rule links with '$(LINK.c) $^', so every prerequisite reaches the compiler driver. A header argument makes clang fail with "cannot specify -o when generating multiple output files". Filter headers out of the link command while retaining them as prerequisites. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-2-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 7ea5ba1df29e..bfb788ea6561 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -257,9 +257,11 @@ $(OUTPUT)/%.o: %.c $(call msg,CC,,$@) $(Q)$(CC) $(CFLAGS) -c $(filter %.c,$^) $(LDLIBS) -o $@ +# Keep header prerequisites off the link command: clang treats +# bare headers as precompiled-header jobs and refuses -o. $(OUTPUT)/%:%.c $(call msg,BINARY,,$@) - $(Q)$(LINK.c) $^ $(LDLIBS) -o $@ + $(Q)$(LINK.c) $(filter-out %.h,$^) $(LDLIBS) -o $@ # LLVM's ld.lld doesn't support all the architectures, so use it only on x86 ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 riscv)) From 35ac30e71745f3fe3f68eaac80c6df55d17ba16c Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:48 -0700 Subject: [PATCH 180/660] selftests/bpf: Drop stale lines, restore two header dependencies Four target-specific lines name objects nothing builds. Commit afef88e65554 ("selftests/bpf: Store BPF object files with .bpf.o extension") left three naming the old BPF objects; flow_dissector_load.o names an intermediate the one-step compile and link rule does not produce. The xsk_xdp_progs, xdp_hw_metadata and xdp_features dependency-map entries are unused: none is listed in LINKED_SKELS, and the regular skeleton rule does not consult the map. The test_l4lb_noinline and test_xdp_noinline '-fno-inline' settings had already stopped taking effect with commit 74b5a5968fe8 ("selftests/bpf: Replace test_progs and test_maps w/ general rule"). The compile recipe uses TRUNNER_BPF_CFLAGS, a simply-expanded copy of BPF_CFLAGS that the target-specific additions cannot affect. The intended functions already carry noinline annotations; restoring the flag produces byte-identical objects. Restore flow_dissector_load.h as a prerequisite of the binary, which is compiled and linked directly from its .c file. Move cgroup_getset_retval_hooks.h under progs/, where the BPF rules' blanket header prerequisite tracks it. Its userspace consumer remains tracked by compiler-generated dependencies. Drop the redundant CURDIR assignment and the unused OBJCOPY definition. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-3-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 14 +------------- .../bpf/prog_tests/cgroup_getset_retval.c | 2 +- .../bpf/{ => progs}/cgroup_getset_retval_hooks.h | 0 3 files changed, 2 insertions(+), 14 deletions(-) rename tools/testing/selftests/bpf/{ => progs}/cgroup_getset_retval_hooks.h (100%) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index bfb788ea6561..3606ee8ab794 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -3,9 +3,6 @@ include ../../../build/Build.include include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include -OBJCOPY ?= $(CROSS_COMPILE)objcopy - -CURDIR := $(abspath .) TOOLSDIR := $(abspath ../../..) LIBDIR := $(TOOLSDIR)/lib BPFDIR := $(LIBDIR)/bpf @@ -339,7 +336,7 @@ $(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS) $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) -$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) +$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) flow_dissector_load.h $(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) $(OUTPUT)/xsk.o: $(BPFOBJ) @@ -477,12 +474,6 @@ BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) -$(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline -$(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline - -$(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h -$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h - # Build BPF object using Clang # $1 - input .c file # $2 - output .o file @@ -543,9 +534,6 @@ linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o -xsk_xdp_progs.skel.h-deps := xsk_xdp_progs.bpf.o -xdp_hw_metadata.skel.h-deps := xdp_hw_metadata.bpf.o -xdp_features.skel.h-deps := xdp_features.bpf.o tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c index 2bb5773d6f99..ea342859d6ce 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c @@ -465,7 +465,7 @@ struct exposed_hook { .expected_err = EXPECTED_ERR, \ }, -#include "cgroup_getset_retval_hooks.h" +#include "progs/cgroup_getset_retval_hooks.h" #undef BPF_RETVAL_HOOK }; diff --git a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h similarity index 100% rename from tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h rename to tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h From 00c2b472d5dd3a9b6ae8b5ae1a250bc21fb84270 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:49 -0700 Subject: [PATCH 181/660] selftests/bpf: Factor the permissive-mode skip suffix into a helper With BPF_STRICT_BUILD=0, eleven recipes append the same "|| { remove the target, print a SKIP marker, report success }" tail, each spelled out inline. Factor the tail into skip_on_fail; every call site keeps its exact message and behavior. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-4-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 49 +++++++++++----------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 3606ee8ab794..11bd8997aea5 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -46,6 +46,14 @@ SKIP_CRYPTO ?= BPF_STRICT_BUILD ?= 1 PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) +# Permissive-mode recipe suffix: on failure, remove the target, emit a +# SKIP marker and report success so the rest of the build continues. +# $(1) - SKIP tag (BPF, TEST, BENCH, SKEL, LINK, ...) +# $(2) - optional extra files to remove alongside the target +# $(3) - optional note appended to the SKIP message +skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) + ifeq ($(srctree),) srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) @@ -481,26 +489,22 @@ CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) # $4 - binary name define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(call skip_on_fail,BPF) endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(call skip_on_fail,BPF) endef # Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4 define CLANG_CPUV4_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(call skip_on_fail,BPF) endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE $(call msg,GCC-BPF,$4,$2) - $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) + $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(call skip_on_fail,BPF) endef SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c @@ -628,10 +632,7 @@ $(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) $$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) && \ diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) && \ $$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ + $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) && \ rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) $(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) @@ -645,10 +646,7 @@ $(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ + $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $$(call skip_on_fail,SKEL) && \ rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) $(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) | $(TRUNNER_OUTPUT) @@ -662,10 +660,7 @@ $(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ + $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $$(call skip_on_fail,SKEL) && \ rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) $(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% @@ -687,10 +682,7 @@ $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TR diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) && \ printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2 && \ $$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ + $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) && \ rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) # When the compiler generates a %.d file, only skel basenames (not @@ -723,8 +715,7 @@ $(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \ $(TRUNNER_TESTS_DIR)/%.c \ | $(TRUNNER_OUTPUT)/%.test.d $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),|| \ - ($(RM) $$@; printf ' %-12s %s\n' 'SKIP-TEST' '$$(notdir $$@)' 1>&2))) + $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$$(call skip_on_fail,TEST)) $$(if $$(TEST_NEEDS_BTFIDS), \ $(Q)if [ -f $$@ ]; then \ $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(TRUNNER_BINARY)]" "$$(notdir $$@)"); \ @@ -960,8 +951,7 @@ $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) # Benchmark runner $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-BENCH' '$(notdir $@)' 1>&2)) + $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(call skip_on_fail,BENCH) $(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h $(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ @@ -1014,8 +1004,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/usdt_2.o \ # $(call msg,BINARY,,$@) - $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-LINK' '$(notdir $@) (some benchmarks may have been skipped)' 1>&2)) + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(call skip_on_fail,LINK,,(some benchmarks may have been skipped)) # This works around GCC warning about snprintf truncating strings like: # From 73fb8cea7269334df6d2996b3a2a09c56d94e4e7 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:50 -0700 Subject: [PATCH 182/660] selftests/bpf: Generate the signing key and certificate once genkey produces the private key and certificate together, but an ordinary multi-target rule can run it twice concurrently when both outputs are required. Only the certificate is currently a prerequisite; the runner split will require both. Use an implicitly grouped pattern rule, as test_kmods already does, to support make versions before 4.3. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-5-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 11bd8997aea5..5bcb762dc3fc 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -783,7 +783,10 @@ VERIFY_SIG_HDR := verification_cert.h VERIFICATION_CERT := $(BUILD_DIR)/signing_key.der PRIVATE_KEY := $(BUILD_DIR)/signing_key.pem -$(VERIFICATION_CERT) $(PRIVATE_KEY): $(VERIFY_SIG_SETUP) +# genkey writes both outputs. Pattern rules group them even on make +# versions before 4.3, preventing concurrent genkey invocations. +# Restrict the pattern to *_key files because genkey uses fixed names. +$(BUILD_DIR)/%_key.pem $(BUILD_DIR)/%_key.der: $(VERIFY_SIG_SETUP) $(Q)mkdir -p $(BUILD_DIR) $(Q)$(VERIFY_SIG_SETUP) genkey $(BUILD_DIR) From 7292e0ad85d5ff25be81dcb1d2625a024f3a8542 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:51 -0700 Subject: [PATCH 183/660] selftests/bpf: Generate verifier/tests.h in a regular recipe The verifier/tests.h recipe is a $(shell ...) expansion: the command runs while make expands the recipe line - including under make -n - its exit status is discarded, and the resulting (empty) expansion is what make actually executes. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-6-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 5bcb762dc3fc..0e23cb506777 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -910,12 +910,12 @@ $(eval $(call DEFINE_TEST_RUNNER,test_maps)) # them (e.g., test.h is using completely pattern), that it's worth just # explicitly defining all the rules explicitly. verifier/tests.h: verifier/*.c - $(shell ( cd verifier/; \ + $(Q)( cd verifier/; \ echo '/* Generated header, do not edit */'; \ echo '#ifdef FILL_ARRAY'; \ ls *.c 2> /dev/null | sed -e 's@\(.*\)@#include \"\1\"@'; \ echo '#endif' \ - ) > verifier/tests.h) + ) > verifier/tests.h $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ From 063fa645c723c188265a01a3ca8c58df6907d6e1 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:52 -0700 Subject: [PATCH 184/660] selftests/bpf: Derive the bench object list from the sources bench links every benchs/bench_*.c object. Derive the object list with a wildcard so adding a benchmark no longer requires updating the link rule. Skeleton prerequisites remain explicit. The sorted list changes object link order and the binary's symbol layout; no benchmark behaves differently. Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-7-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 0e23cb506777..99237c3b32d5 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -951,7 +951,8 @@ $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) $(call msg,CXX,,$@) $(Q)$(CXX) $(CXXFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@ -# Benchmark runner +# Benchmark runner. Every benchs/bench_*.c is compiled and linked in. +BENCH_OBJS := $(sort $(patsubst benchs/%.c,$(OUTPUT)/%.o,$(wildcard benchs/bench_*.c))) $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) $(call msg,CC,,$@) $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(call skip_on_fail,BENCH) @@ -982,30 +983,9 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(TESTING_HELPERS) \ $(TRACE_HELPERS) \ $(CGROUP_HELPERS) \ - $(OUTPUT)/bench_count.o \ - $(OUTPUT)/bench_rename.o \ - $(OUTPUT)/bench_trigger.o \ - $(OUTPUT)/bench_ringbufs.o \ - $(OUTPUT)/bench_bloom_filter_map.o \ - $(OUTPUT)/bench_bpf_loop.o \ - $(OUTPUT)/bench_bpf_for.o \ - $(OUTPUT)/bench_strncmp.o \ - $(OUTPUT)/bench_bpf_hashmap_full_update.o \ - $(OUTPUT)/bench_local_storage.o \ - $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o \ - $(OUTPUT)/bench_bpf_hashmap_lookup.o \ - $(OUTPUT)/bench_local_storage_create.o \ - $(OUTPUT)/bench_htab_mem.o \ - $(OUTPUT)/bench_bpf_crypto.o \ - $(OUTPUT)/bench_sockmap.o \ - $(OUTPUT)/bench_lpm_trie_map.o \ - $(OUTPUT)/bench_bpf_timing.o \ - $(OUTPUT)/bench_bpf_nop.o \ - $(OUTPUT)/bench_xdp_lb.o \ - $(OUTPUT)/bench_libarena.o \ + $(BENCH_OBJS) \ $(OUTPUT)/usdt_1.o \ - $(OUTPUT)/usdt_2.o \ - # + $(OUTPUT)/usdt_2.o $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(call skip_on_fail,LINK,,(some benchmarks may have been skipped)) From da6cb3aff2cb746164eff8be8c14e7b7a0b1e430 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:53 -0700 Subject: [PATCH 185/660] selftests/bpf: Extract BPF skeleton generation into a helper script Four skeleton recipes duplicate the link, determinism-check and header-generation pipeline inside DEFINE_TEST_RUNNER_RULES. Extract it into gen_bpf_skel.sh, with options for light and signed skeletons. Intermediate names now derive from the output header, retaining separate linked/llinked infixes for regular and light skeletons. Keep the permissive-mode missing-input guards in a skip_if_missing helper, so a skipped skeleton still prints only SKIP-SKEL. On failure, the script removes intermediates and both output headers; previously, strict-mode recipes left intermediates behind and .DELETE_ON_ERROR covered only the target, not its subskeleton. The determinism check names the skeleton when it fails, and linked skeletons log GEN-SKEL before linking. Suggested-by: Eduard Zingerman Signed-off-by: Mykola Lysenko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260921075855.2065871-8-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 89 ++++++++---------- tools/testing/selftests/bpf/gen_bpf_skel.sh | 99 +++++++++++++++++++++ 2 files changed, 134 insertions(+), 54 deletions(-) create mode 100755 tools/testing/selftests/bpf/gen_bpf_skel.sh diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 99237c3b32d5..a6fd107e357f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -54,6 +54,17 @@ PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) +# Permissive-mode recipe prefix: when a prerequisite object is missing +# (its compile failed and was skipped), remove the target and $(3), +# emit a SKIP marker and report success without running the rest of +# the recipe. +# $(1) - SKIP tag +# $(2) - input files to check +# $(3) - optional extra files to remove alongside the target +skip_if_missing = $(if $(PERMISSIVE),for f in $(2); do [ -f $$f ] || { \ + $(RM) $@ $(3); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)' 1>&2; exit 0; }; done;) + ifeq ($(srctree),) srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) @@ -560,7 +571,6 @@ HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ # $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) define DEFINE_TEST_RUNNER -LSKEL_SIGN := -S -k $(PRIVATE_KEY) -i $(VERIFICATION_CERT) TRUNNER_OUTPUT := $(OUTPUT)$(if $2,/)$2 TRUNNER_BINARY := $1$(if $2,-)$2 TRUNNER_TEST_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.test.o, \ @@ -620,70 +630,41 @@ $(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ $$($$<-CFLAGS) \ $$($$<-$2-CFLAGS),$(TRUNNER_BINARY)) -$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ +$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) + $(Q)$$(call skip_if_missing,SKEL,$$<,$$(@:.skel.h=.subskel.h)) \ printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) && \ - diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) && \ - $$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) && \ - rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) + BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ + --name $$(notdir $$(<:.bpf.o=)) \ + --skel $$@ --subskel $$(@:.skel.h=.subskel.h) $$< $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) -$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ +$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) + $(Q)$$(call skip_if_missing,SKEL,$$<) \ printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $$(call skip_on_fail,SKEL) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) + BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ + --name $$(notdir $$(<:.bpf.o=_lskel)) --lskel \ + --skel $$@ $$< $$(call skip_on_fail,SKEL) -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ +$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) + $(Q)$$(call skip_if_missing,SKEL,$$<) \ printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY) (signed)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $$(call skip_on_fail,SKEL) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) + BPFTOOL=$$(BPFTOOL) PRIVATE_KEY=$(PRIVATE_KEY) \ + VERIFICATION_CERT=$(VERIFICATION_CERT) \ + ./gen_bpf_skel.sh --sign \ + --name $$(notdir $$(<:.bpf.o=_lskel)) \ + --skel $$@ $$< $$(call skip_on_fail,SKEL) $(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% # .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites .SECONDEXPANSION: -$(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),for f in $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)); do \ - if [ ! -f $$$$f ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi; \ - done;) \ +$(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) + $(Q)$$(call skip_if_missing,SKEL,$$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)),$$(@:.skel.h=.subskel.h)) \ printf ' %-12s %s\n' 'LINK-BPF' '[$(TRUNNER_BINARY)] $$(notdir $$(@:.skel.h=.bpf.o))' 1>&2; \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) && \ - diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) && \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2 && \ - $$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) && \ - rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) + printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ + BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ + --name $$(notdir $$(@:.skel.h=)) \ + --skel $$@ --subskel $$(@:.skel.h=.subskel.h) \ + $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) # When the compiler generates a %.d file, only skel basenames (not # full paths) are specified as prerequisites for corresponding %.o diff --git a/tools/testing/selftests/bpf/gen_bpf_skel.sh b/tools/testing/selftests/bpf/gen_bpf_skel.sh new file mode 100755 index 000000000000..60942f24b094 --- /dev/null +++ b/tools/testing/selftests/bpf/gen_bpf_skel.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Link BPF object file(s) with "bpftool gen object" and generate a +# skeleton (or light skeleton) header from the result. +# +# Usage: +# BPFTOOL= gen_bpf_skel.sh --name NAME --skel OUT +# [options] OBJ... +# +# --name NAME skeleton object name ("name NAME" for bpftool) +# --skel OUT output header ("foo.skel.h" or "foo.lskel.h") +# --subskel OUT also generate a subskeleton header into OUT +# --lskel generate a light skeleton (bpftool gen skeleton -L) +# --sign sign the (light) skeleton, with the key and +# certificate taken from $PRIVATE_KEY and +# $VERIFICATION_CERT +# +# The inputs are linked three times and the 2nd and 3rd results compared, +# as a regression test for the determinism of "bpftool gen object". +# +# Intermediate files are named after the output header with a "linked" +# infix - "llinked" for light or signed skeletons, so that generating a +# .skel.h and a .lskel.h from the same .bpf.o in parallel never races on +# the intermediates. +# +# The bpftool binary is taken from $BPFTOOL (default: bpftool from PATH). +# On failure all outputs and intermediates are removed and the script exits +# non-zero; permissive-mode skipping is the caller's business (see +# skip_on_fail in the Makefile). + +set -u + +bpftool=${BPFTOOL:-bpftool} +name='' skel='' subskel='' +lskel=0 sign=0 + +while [ $# -gt 0 ]; do + case "$1" in + --name) name=$2; shift 2 ;; + --skel) skel=$2; shift 2 ;; + --subskel) subskel=$2; shift 2 ;; + --lskel) lskel=1; shift ;; + --sign) sign=1; shift ;; + --) shift; break ;; + -*) echo "$0: unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ -z "$name" ] || [ -z "$skel" ] || [ $# -eq 0 ]; then + echo "usage: $0 --name NAME --skel OUT [options] OBJ..." >&2 + exit 1 +fi + +infix=linked +if [ "$lskel" -eq 1 ] || [ "$sign" -eq 1 ]; then + infix=llinked +fi + +if [ "$sign" -eq 1 ] && { [ -z "${PRIVATE_KEY-}" ] || [ -z "${VERIFICATION_CERT-}" ]; }; then + echo "$0: --sign requires PRIVATE_KEY and VERIFICATION_CERT in the environment" >&2 + exit 1 +fi + +base=${skel%.skel.h} +base=${base%.lskel.h} +t1=$base.${infix}1.o +t2=$base.${infix}2.o +t3=$base.${infix}3.o + +fail() { + rm -f "$skel" ${subskel:+"$subskel"} "$t1" "$t2" "$t3" + exit 1 +} + +"$bpftool" gen object "$t1" "$@" || fail +"$bpftool" gen object "$t2" "$t1" || fail +"$bpftool" gen object "$t3" "$t2" || fail +if ! cmp -s "$t2" "$t3"; then + echo "$0: bpftool gen object is not deterministic for $skel" >&2 + fail +fi + +args=() +if [ "$sign" -eq 1 ]; then + args+=(-S -k "$PRIVATE_KEY" -i "$VERIFICATION_CERT") +fi +if [ "$lskel" -eq 1 ]; then + args+=(-L) +fi +"$bpftool" gen skeleton ${args[@]+"${args[@]}"} "$t3" name "$name" > "$skel" || fail + +if [ -n "$subskel" ]; then + "$bpftool" gen subskeleton "$t3" name "$name" > "$subskel" || fail +fi + +rm -f "$t1" "$t2" "$t3" +exit 0 From ab1d791d0cc4e986738e3157fb591dc0c3e23235 Mon Sep 17 00:00:00 2001 From: Mykola Lysenko Date: Mon, 21 Sep 2026 00:58:54 -0700 Subject: [PATCH 186/660] selftests/bpf: Move shared build definitions into Makefile.buildvars Move shared build definitions into Makefile.buildvars, preserving their order, in preparation for separate runner sub-makes. Include it after ../lib.mk and before Makefile.feature, so the LLVM feature probe still sees srctree. Keep the CFLAGS and LDFLAGS additions before lib.mk to preserve flag ordering; their references to definitions below the include expand when used. Name the clang warning suppression CLANG_WARN_CFLAGS so it can be referenced there. The arena-ASAN probe moves below lib.mk, so it queries the CLANG the BPF objects are built with. The CPU-v4 probe, BPF_GCC and TEST_KMODS remain before lib.mk because they determine the target lists. Definitions no runner reads move too when they are declared beside ones a runner does. BPFTOOLDIR, HOST_BPFOBJ and BPF_TARGET_ENDIAN stay in the Makefile, which alone reads them, and follow the include because their := assignments read values it defines. Suggested-by: Eduard Zingerman Signed-off-by: Mykola Lysenko Link: https://lore.kernel.org/bpf/20260921075855.2065871-9-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 151 ++---------------- .../testing/selftests/bpf/Makefile.buildvars | 150 +++++++++++++++++ 2 files changed, 160 insertions(+), 141 deletions(-) create mode 100644 tools/testing/selftests/bpf/Makefile.buildvars diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index a6fd107e357f..25a21247a1ba 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -3,109 +3,21 @@ include ../../../build/Build.include include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include -TOOLSDIR := $(abspath ../../..) -LIBDIR := $(TOOLSDIR)/lib -BPFDIR := $(LIBDIR)/bpf -TOOLSINCDIR := $(TOOLSDIR)/include -TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include -BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool -APIDIR := $(TOOLSINCDIR)/uapi -ifneq ($(O),) -GENDIR := $(O)/include/generated -else -GENDIR := $(abspath ../../../../include/generated) -endif -GENHDR := $(GENDIR)/autoconf.h -PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config - -ifneq ($(wildcard $(GENHDR)),) - GENFLAGS := -DHAVE_GENHDR -endif - BPF_GCC ?= $(shell command -v bpf-gcc;) -ifdef ASAN -SAN_CFLAGS ?= -fsanitize=address -fno-omit-frame-pointer -else -SAN_CFLAGS ?= -endif -SAN_LDFLAGS ?= $(SAN_CFLAGS) -RELEASE ?= -OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) - -LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null) -LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) SKIP_DOCS ?= SKIP_LLVM ?= SKIP_LIBBFD ?= SKIP_CRYPTO ?= -# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or -# benchmark compilation failure is fatal. Set to 0 to tolerate failures -# and continue building the remaining tests. -BPF_STRICT_BUILD ?= 1 -PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) - -# Permissive-mode recipe suffix: on failure, remove the target, emit a -# SKIP marker and report success so the rest of the build continues. -# $(1) - SKIP tag (BPF, TEST, BENCH, SKEL, LINK, ...) -# $(2) - optional extra files to remove alongside the target -# $(3) - optional note appended to the SKIP message -skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ - printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) - -# Permissive-mode recipe prefix: when a prerequisite object is missing -# (its compile failed and was skipped), remove the target and $(3), -# emit a SKIP marker and report success without running the rest of -# the recipe. -# $(1) - SKIP tag -# $(2) - input files to check -# $(3) - optional extra files to remove alongside the target -skip_if_missing = $(if $(PERMISSIVE),for f in $(2); do [ -f $$f ] || { \ - $(RM) $@ $(3); \ - printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)' 1>&2; exit 0; }; done;) - -ifeq ($(srctree),) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ - -Wall -Werror -fno-omit-frame-pointer \ - -Wno-unused-but-set-variable \ - $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ - -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ - -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ - -I$(CURDIR)/libarena/include -LDFLAGS += $(SAN_LDFLAGS) -LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread - -PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") -PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) -LDLIBS += $(PCAP_LIBS) -CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) - # Some utility functions use LLVM libraries jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) -ifneq ($(LLVM),) -# Silence some warnings when compiled with clang -CFLAGS += -Wno-unused-command-line-argument -endif - # Check whether bpf cpu=v4 is supported or not by clang ifneq ($(shell $(CLANG) --target=bpf -mcpu=help 2>&1 | grep 'v4'),) CLANG_CPUV4 := 1 endif -# Check whether clang supports BPF address sanitizer (requires LLVM 22+) -CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ - $(CLANG) --target=bpf -fsanitize=kernel-address \ - -mllvm -asan-shadow-addr-space=1 \ - -x c -c - -o /dev/null 2>/dev/null && echo 1) - # Order correspond to 'make run_tests' order TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_progs \ test_sockmap \ @@ -183,7 +95,17 @@ override define CLEAN $(Q)$(MAKE) docs-clean endef +# Append build flags before lib.mk's user flags. References to +# Makefile.buildvars definitions expand when the flags are used. +# The arena-ASAN define is appended after lib.mk. +CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) $(CLANG_WARN_CFLAGS) +LDFLAGS += $(SAN_LDFLAGS) include ../lib.mk +include Makefile.buildvars + +BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool +HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a +BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) NON_CHECK_FEAT_TARGETS := clean docs-clean emit_tests CHECK_FEAT := $(filter-out $(NON_CHECK_FEAT_TARGETS),$(or $(MAKECMDGOALS), "none")) @@ -228,21 +150,6 @@ ifeq ($(feature-llvm),1) endif endif -SCRATCH_DIR := $(OUTPUT)/tools -BUILD_DIR := $(SCRATCH_DIR)/build -INCLUDE_DIR := $(SCRATCH_DIR)/include -BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a -ifneq ($(CROSS_COMPILE),) -HOST_BUILD_DIR := $(BUILD_DIR)/host -HOST_SCRATCH_DIR := $(OUTPUT)/host-tools -HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include -else -HOST_BUILD_DIR := $(BUILD_DIR) -HOST_SCRATCH_DIR := $(SCRATCH_DIR) -HOST_INCLUDE_DIR := $(INCLUDE_DIR) -endif -HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a -RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ ../../../../vmlinux \ @@ -330,17 +237,6 @@ $(TEST_KMOD_TARGETS): $(addprefix test_kmods/,$(TEST_KMODS)) $(call msg,MOD,,$@) $(Q)$(if $(PERMISSIVE),if [ -f test_kmods/$(@F) ]; then )cp test_kmods/$(@F) $@$(if $(PERMISSIVE),; fi) - -DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -ifneq ($(CROSS_COMPILE),) -CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool -TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) -USE_BOOTSTRAP := "" -else -TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) -USE_BOOTSTRAP := "bootstrap/" -endif - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -360,7 +256,6 @@ $(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) $(OUTPUT)/xsk.o: $(BPFOBJ) -BPFTOOL ?= $(DEFAULT_BPFTOOL) $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ @@ -467,30 +362,11 @@ $(shell $(1) $(2) -dM -E - /dev/null) +LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) + +# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or +# benchmark compilation failure is fatal. Set to 0 to tolerate failures +# and continue building the remaining tests. +BPF_STRICT_BUILD ?= 1 +PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) + +# Permissive-mode recipe suffix: on failure, remove the target, emit a +# SKIP marker and report success so the rest of the build continues. +# $(1) - SKIP tag (BPF, TEST, BENCH, SKEL, LINK, ...) +# $(2) - optional extra files to remove alongside the target +# $(3) - optional note appended to the SKIP message +skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) + +# Permissive-mode recipe prefix: when a prerequisite object is missing +# (its compile failed and was skipped), remove the target and $(3), +# emit a SKIP marker and report success without running the rest of +# the recipe. +# $(1) - SKIP tag +# $(2) - input files to check +# $(3) - optional extra files to remove alongside the target +skip_if_missing = $(if $(PERMISSIVE),for f in $(2); do [ -f $$f ] || { \ + $(RM) $@ $(3); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)' 1>&2; exit 0; }; done;) + +ifeq ($(srctree),) +srctree := $(patsubst %/,%,$(dir $(CURDIR))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +endif + +COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ + -Wall -Werror -fno-omit-frame-pointer \ + -Wno-unused-but-set-variable \ + $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ + -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ + -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ + -I$(CURDIR)/libarena/include +LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread + +PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") +PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) +LDLIBS += $(PCAP_LIBS) + +ifneq ($(LLVM),) +# Silence some warnings when compiled with clang +CLANG_WARN_CFLAGS := -Wno-unused-command-line-argument +endif + +# Check whether clang supports BPF address sanitizer (requires LLVM 22+). +# This runs after ../lib.mk has resolved CLANG from LLVM=, +# so it probes the compiler that builds the BPF objects. +CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ + $(CLANG) --target=bpf -fsanitize=kernel-address \ + -mllvm -asan-shadow-addr-space=1 \ + -x c -c - -o /dev/null 2>/dev/null && echo 1) + +SCRATCH_DIR := $(OUTPUT)/tools +BUILD_DIR := $(SCRATCH_DIR)/build +INCLUDE_DIR := $(SCRATCH_DIR)/include +BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a +ifneq ($(CROSS_COMPILE),) +HOST_BUILD_DIR := $(BUILD_DIR)/host +HOST_SCRATCH_DIR := $(OUTPUT)/host-tools +HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include +else +HOST_BUILD_DIR := $(BUILD_DIR) +HOST_SCRATCH_DIR := $(SCRATCH_DIR) +HOST_INCLUDE_DIR := $(INCLUDE_DIR) +endif +RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids + +DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool +ifneq ($(CROSS_COMPILE),) +CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool +TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) +USE_BOOTSTRAP := "" +else +TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) +USE_BOOTSTRAP := "bootstrap/" +endif + +BPFTOOL ?= $(DEFAULT_BPFTOOL) + +# Determine target endianness. +IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - Date: Mon, 21 Sep 2026 00:58:55 -0700 Subject: [PATCH 187/660] selftests/bpf: Build each test runner instance in its own sub-make Replace DEFINE_TEST_RUNNER/DEFINE_TEST_RUNNER_RULES with Makefile.runner, invoked once per test runner instance. Each invocation uses ordinary make rules in a single-flavor namespace. The main Makefile owns shared build outputs and the kselftest run/install rules. Makefile.skel supplies the BPF object and skeleton rules used by both the main Makefile and the runners. The main Makefile builds the default flavor before invoking unflavored runners; flavored runners build in separate directories. Build shared userspace objects once and link them into every flavor. Order their compilation after the bpftool sub-build, which installs the libbpf-internal headers they include. Pass the assembled CFLAGS and LDFLAGS to runners on their command lines. Build and copy runtime fixtures alongside runner compilation so module builds do not delay runner startup. They are prerequisites of the test_progs, test_progs- and all goals rather than of the runner binaries, so a goal spelled as a binary's path builds the binary alone. Installation copies the default flavor's BPF objects, preserving the previous result. Generate prog_tests/tests.h and map_tests/tests.h through ordinary recipes, avoiding generation during make -n. Signed skeletons explicitly depend on the private key. The flavored runners' objects and the test objects are targets of the sub-makes only, and the bare linked-object names (make linked_funcs1.bpf.o) are no longer targets. Build-log details change: skeleton messages adopt the common format and output stream, the shared objects log as CC, the fixture copy prints no EXT-COPY line and TEST-HDR carries no runner tag. Co-developed-by: Eduard Zingerman Signed-off-by: Eduard Zingerman Signed-off-by: Mykola Lysenko Link: https://lore.kernel.org/bpf/20260921075855.2065871-10-nickolay.lysenko@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 497 ++++++------------ .../testing/selftests/bpf/Makefile.buildvars | 16 +- tools/testing/selftests/bpf/Makefile.runner | 125 +++++ tools/testing/selftests/bpf/Makefile.skel | 139 +++++ 4 files changed, 426 insertions(+), 351 deletions(-) create mode 100644 tools/testing/selftests/bpf/Makefile.runner create mode 100644 tools/testing/selftests/bpf/Makefile.skel diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 25a21247a1ba..afa589a27b15 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -10,9 +10,6 @@ SKIP_LLVM ?= SKIP_LIBBFD ?= SKIP_CRYPTO ?= -# Some utility functions use LLVM libraries -jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) - # Check whether bpf cpu=v4 is supported or not by clang ifneq ($(shell $(CLANG) --target=bpf -mcpu=help 2>&1 | grep 'v4'),) CLANG_CPUV4 := 1 @@ -29,16 +26,6 @@ TEST_INST_SUBDIRS := no_alu32 ifneq ($(BPF_GCC),) TEST_GEN_PROGS += test_progs-bpf_gcc TEST_INST_SUBDIRS += bpf_gcc - -# The following tests contain C code that, although technically legal, -# triggers GCC warnings that cannot be disabled: declaration of -# anonymous struct types in function parameter lists. -progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error - endif ifneq ($(CLANG_CPUV4),) @@ -150,6 +137,9 @@ ifeq ($(feature-llvm),1) endif endif +# Some utility functions use LLVM libraries +$(OUTPUT)/jit_disasm_helpers.o: CFLAGS += $(LLVM_CFLAGS) + VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ ../../../../vmlinux \ @@ -168,7 +158,7 @@ $(notdir $(TEST_GEN_PROGS) $(TEST_KMODS) \ $(TEST_GEN_PROGS_EXTENDED)): %: $(OUTPUT)/% ; # sort removes libbpf duplicates when not cross-building -MAKE_DIRS := $(sort $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ +MAKE_DIRS := $(sort $(OUTPUT) $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ $(BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/bpftool \ $(HOST_BUILD_DIR)/resolve_btfids \ $(INCLUDE_DIR)) @@ -252,9 +242,27 @@ $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELP $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) $(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) flow_dissector_load.h -$(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) -$(OUTPUT)/xsk.o: $(BPFOBJ) + +# Defined ahead of the first rule that lists it as a prerequisite. +VERIFY_SIG_HDR := verification_cert.h + +# The userspace objects the runner instances link (RUNNER_OBJS, see +# Makefile.buildvars) are built here, once, so that no runner sub-make +# ever writes a file this Makefile also builds. +# +# Helper C files include headers installed to $(INCLUDE_DIR) by the +# $(TRUNNER_BPFTOOL) and $(BPFOBJ) targets. +RUNNER_OBJS := $(sort $(RUNNER_OBJS-test_progs) $(RUNNER_OBJS-test_maps)) +$(RUNNER_OBJS): $(VERIFY_SIG_HDR) $(BPFOBJ) $(TRUNNER_BPFTOOL) +$(OUTPUT)/test_progs.o: prog_tests/tests.h +$(OUTPUT)/test_maps.o: map_tests/tests.h + +# find_bit.c is the only runner source outside this directory, so it +# needs a rule of its own. +$(OUTPUT)/find_bit.o: $(TOOLSDIR)/lib/find_bit.c + $(call msg,CC,,$@) + $(Q)$(CC) $(CFLAGS) -c $< $(LDLIBS) -o $@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool @@ -323,7 +331,7 @@ endif # vmlinux.h is first dumped to a temporary file and then compared to # the previous version. This helps to avoid unnecessary re-builds of -# $(TRUNNER_BPF_OBJS) +# BPF objects. $(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) ifeq ($(VMLINUX_H),) $(call msg,GEN,,$@) @@ -369,274 +377,6 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) -# Build BPF object using Clang -# $1 - input .c file -# $2 - output .o file -# $3 - CFLAGS -# $4 - binary name -define CLANG_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(call skip_on_fail,BPF) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 -define CLANG_NOALU32_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(call skip_on_fail,BPF) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4 -define CLANG_CPUV4_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(call skip_on_fail,BPF) -endef -# Build BPF object using GCC -define GCC_BPF_BUILD_RULE - $(call msg,GCC-BPF,$4,$2) - $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(call skip_on_fail,BPF) -endef - -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c - -LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ - linked_vars.skel.h linked_maps.skel.h \ - test_subskeleton.skel.h test_subskeleton_lib.skel.h \ - test_usdt.skel.h tracing_multi.skel.h \ - tracing_multi_module.skel.h \ - tracing_multi_intersect.skel.h \ - tracing_multi_session.skel.h - -LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ - core_kern.c core_kern_overflow.c test_ringbuf.c \ - test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ - test_ringbuf_overwrite.c - -LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c - -# Generate both light skeleton and libbpf skeleton for these -LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c test_global_percpu_data.c -SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) - -test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o -linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o -linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o -linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o -# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file -# but that's created as a side-effect of the skel.h generation. -test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o -test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o -test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o -tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o -tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o - -LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) -LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) - -HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ - $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ - $(addprefix $(BPFDIR)/, bpf_core_read.h \ - bpf_endian.h \ - bpf_helpers.h \ - bpf_tracing.h) - -# Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on -# $eval()) and pass control to DEFINE_TEST_RUNNER_RULES. -# Parameters: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER - -TRUNNER_OUTPUT := $(OUTPUT)$(if $2,/)$2 -TRUNNER_BINARY := $1$(if $2,-)$2 -TRUNNER_TEST_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.test.o, \ - $$(notdir $$(wildcard $(TRUNNER_TESTS_DIR)/*.c))) -TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_EXTRA_SOURCES))) -TRUNNER_LIB_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_LIB_SOURCES))) -TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES)) -TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h -TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c)) -TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS)) -TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \ - $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\ - $$(TRUNNER_BPF_SRCS))) -TRUNNER_BPF_LSKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS) $$(LSKELS_EXTRA)) -TRUNNER_BPF_SKELS_LINKED := $$(addprefix $$(TRUNNER_OUTPUT)/,$(LINKED_SKELS)) -TRUNNER_BPF_LSKELS_SIGNED := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS_SIGNED)) -TEST_GEN_FILES += $$(TRUNNER_BPF_OBJS) - -# Evaluate rules now with extra TRUNNER_XXX variables above already defined -$$(eval $$(call DEFINE_TEST_RUNNER_RULES,$1,$2)) - -endef - -# Using TRUNNER_XXX variables, provided by callers of DEFINE_TEST_RUNNER and -# set up by DEFINE_TEST_RUNNER itself, create test runner build rules with: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER_RULES - -# Permissive build behaviour (skip-on-failure compile, partial-link) only -# applies to test_progs and its flavors; runners that use strong cross-object -# references (e.g. test_maps) keep strict semantics even when permissive. -# The check is inlined per-runner so $1 is substituted at $(call) time and -# the result is baked into each rule's recipe. - -ifeq ($($(TRUNNER_OUTPUT)-dir),) -$(TRUNNER_OUTPUT)-dir := y -$(TRUNNER_OUTPUT): - $$(call msg,MKDIR,,$$@) - $(Q)mkdir -p $$@ -endif - -# ensure we set up BPF objects generation rule just once for a given -# input/output directory combination -ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),) -$(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y -$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ - $(TRUNNER_BPF_PROGS_DIR)/%.c \ - $(TRUNNER_BPF_PROGS_DIR)/*.h \ - $$(INCLUDE_DIR)/vmlinux.h \ - $(HEADERS_FOR_BPF_OBJS) \ - | $(TRUNNER_OUTPUT) $$(BPFOBJ) - $$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \ - $(TRUNNER_BPF_CFLAGS) \ - $$($$<-CFLAGS) \ - $$($$<-$2-CFLAGS),$(TRUNNER_BINARY)) - -$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) - $(Q)$$(call skip_if_missing,SKEL,$$<,$$(@:.skel.h=.subskel.h)) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ - --name $$(notdir $$(<:.bpf.o=)) \ - --skel $$@ --subskel $$(@:.skel.h=.subskel.h) $$< $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) - -$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) - $(Q)$$(call skip_if_missing,SKEL,$$<) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ - --name $$(notdir $$(<:.bpf.o=_lskel)) --lskel \ - --skel $$@ $$< $$(call skip_on_fail,SKEL) - -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) - $(Q)$$(call skip_if_missing,SKEL,$$<) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY) (signed)] $$(notdir $$@)' 1>&2; \ - BPFTOOL=$$(BPFTOOL) PRIVATE_KEY=$(PRIVATE_KEY) \ - VERIFICATION_CERT=$(VERIFICATION_CERT) \ - ./gen_bpf_skel.sh --sign \ - --name $$(notdir $$(<:.bpf.o=_lskel)) \ - --skel $$@ $$< $$(call skip_on_fail,SKEL) - -$(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% - -# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites -.SECONDEXPANSION: -$(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) gen_bpf_skel.sh | $(TRUNNER_OUTPUT) - $(Q)$$(call skip_if_missing,SKEL,$$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)),$$(@:.skel.h=.subskel.h)) \ - printf ' %-12s %s\n' 'LINK-BPF' '[$(TRUNNER_BINARY)] $$(notdir $$(@:.skel.h=.bpf.o))' 1>&2; \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - BPFTOOL=$$(BPFTOOL) ./gen_bpf_skel.sh \ - --name $$(notdir $$(@:.skel.h=)) \ - --skel $$@ --subskel $$(@:.skel.h=.subskel.h) \ - $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) $$(call skip_on_fail,SKEL,$$(@:.skel.h=.subskel.h)) - -# When the compiler generates a %.d file, only skel basenames (not -# full paths) are specified as prerequisites for corresponding %.o -# file. vpath directives below instruct make to search for skel files -# in TRUNNER_OUTPUT, if they are not present in the working directory. -vpath %.skel.h $(TRUNNER_OUTPUT) -vpath %.lskel.h $(TRUNNER_OUTPUT) -vpath %.subskel.h $(TRUNNER_OUTPUT) - -endif - -# ensure we set up tests.h header generation rule just once -ifeq ($($(TRUNNER_TESTS_DIR)-tests-hdr),) -$(TRUNNER_TESTS_DIR)-tests-hdr := y -$(TRUNNER_TESTS_HDR): $(TRUNNER_TESTS_DIR)/*.c - $$(call msg,TEST-HDR,$(TRUNNER_BINARY),$$@) - $$(shell (echo '/* Generated header, do not edit */'; \ - sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ - $(TRUNNER_TESTS_DIR)/*.c | sort ; \ - ) > $$@) -endif - -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(TRUNNER_OUTPUT)/btf_data.bpf.o -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 - -# compile individual test files -# Note: we cd into output directory to ensure embedded BPF object is found -$(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \ - $(TRUNNER_TESTS_DIR)/%.c \ - | $(TRUNNER_OUTPUT)/%.test.d - $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$$(call skip_on_fail,TEST)) - $$(if $$(TEST_NEEDS_BTFIDS), \ - $(Q)if [ -f $$@ ]; then \ - $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(TRUNNER_BINARY)]" "$$(notdir $$@)"); \ - $(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@; \ - $(RESOLVE_BTFIDS) --patch_btfids $$@.BTF_ids $$@; \ - fi) - -$(TRUNNER_TEST_OBJS:.o=.d): $(TRUNNER_OUTPUT)/%.test.d: \ - $(TRUNNER_TESTS_DIR)/%.c \ - $(TRUNNER_EXTRA_HDRS) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) \ - $(TRUNNER_BPF_SKELS) \ - $(TRUNNER_BPF_LSKELS) \ - $(TRUNNER_BPF_LSKELS_SIGNED) \ - $(TRUNNER_BPF_SKELS_LINKED) - -ifeq ($(filter clean docs-clean emit_tests,$(MAKECMDGOALS)),) -include $(wildcard $(TRUNNER_TEST_OBJS:.o=.d)) -endif - -# add per extra obj CFGLAGS definitions -$(foreach N,$(patsubst $(TRUNNER_OUTPUT)/%.o,%,$(TRUNNER_EXTRA_OBJS)), \ - $(eval $(TRUNNER_OUTPUT)/$(N).o: CFLAGS += $($(N).c-CFLAGS))) - -$(TRUNNER_EXTRA_OBJS): $(TRUNNER_OUTPUT)/%.o: \ - %.c \ - $(TRUNNER_EXTRA_HDRS) \ - $(VERIFY_SIG_HDR) \ - $(TRUNNER_TESTS_HDR) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) - $$(call msg,EXT-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -$(TRUNNER_LIB_OBJS): $(TRUNNER_OUTPUT)/%.o:$(TOOLSDIR)/lib/%.c - $$(call msg,LIB-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -# non-flavored in-srctree builds receive special treatment, in particular, we -# do not need to copy extra resources (see e.g. test_btf_dump_case()) -$(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) | $(TRUNNER_OUTPUT) -ifneq ($2:$(OUTPUT),:$(shell pwd)) - $$(call msg,EXT-COPY,$(TRUNNER_BINARY),$(TRUNNER_EXTRA_FILES)) - $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) $$^ $(TRUNNER_OUTPUT)/ -endif - -# some X.test.o files have runtime dependencies on Y.bpf.o files -$(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS) - -$(OUTPUT)/$(TRUNNER_BINARY): $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(wildcard $(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)) \ - $(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \ - $(TRUNNER_LIB_OBJS) \ - $(TRUNNER_BPFTOOL) \ - $(OUTPUT)/veristat \ - | $(TRUNNER_BINARY)-extras \ - $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$(TRUNNER_TEST_OBJS))) - $$(call msg,BINARY,,$$@) - $(Q)$$(CC) $$(CFLAGS) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(filter %.a %.o,$$(wildcard $(TRUNNER_TEST_OBJS)) $$(filter-out $(TRUNNER_TEST_OBJS),$$^)),$$(filter %.a %.o,$$^)),$$(filter %.a %.o,$$^)) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@ - $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \ - $(OUTPUT)/$(if $2,$2/)bpftool - -endef - -VERIFY_SIG_HDR := verification_cert.h - # genkey writes both outputs. Pattern rules group them even on make # versions before 4.3, preventing concurrent genkey invocations. # Restrict the pattern to *_key files because genkey uses fixed names. @@ -682,78 +422,137 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# Define test_progs test runner. -TRUNNER_TESTS_DIR := prog_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_progs.c \ - cgroup_helpers.c \ - trace_helpers.c \ - network_helpers.c \ - testing_helpers.c \ - btf_helpers.c \ - cap_helpers.c \ - unpriv_helpers.c \ - sysctl_helpers.c \ - netlink_helpers.c \ - jit_disasm_helpers.c \ - io_helpers.c \ - test_loader.c \ - xsk.c \ - disasm.c \ - disasm_helpers.c \ - json_writer.c \ - $(VERIFY_SIG_HDR) \ - flow_dissector_load.h \ - ip_check_defrag_frags.h \ - bpftool_helpers.c \ - usdt_1.c usdt_2.c \ - $(LIBARENA_SKEL) \ - $(LIBARENA_ASAN_SKEL) -TRUNNER_LIB_SOURCES := find_bit.c -TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ - $(OUTPUT)/liburandom_read.so \ - $(OUTPUT)/xdp_synproxy \ - $(OUTPUT)/sign-file \ - $(OUTPUT)/uprobe_multi \ - $(TEST_KMOD_TARGETS) \ - ima_setup.sh \ - $(VERIFY_SIG_SETUP) \ - $(wildcard progs/btf_dump_test_case_*.c) \ - $(wildcard progs/*.bpf.o) \ - bpftool_btf_dump_sorted.expected \ - bpftool_btf_dump_unsorted.expected -TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs)) +# Generated test list headers -# Define test_progs-no_alu32 test runner. -TRUNNER_BPF_BUILD_RULE := CLANG_NOALU32_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,no_alu32)) +define gen_tests_hdr + $(call msg,TEST-HDR,,$@) + $(Q)(echo '/* Generated header, do not edit */'; \ + sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ + $(@D)/*.c | sort) > $@ +endef + +prog_tests/tests.h: $(wildcard prog_tests/*.c) + $(gen_tests_hdr) + +map_tests/tests.h: $(wildcard map_tests/*.c) + $(gen_tests_hdr) + +# Test runner instances, one sub-make each (see Makefile.runner). + +# Share LLVM probe results with runners. Pass CC explicitly to +# runners so it does not affect libbpf's compiler selection. +export LLVM_LDLIBS LLVM_LDFLAGS + +# Pass assembled flags on the command line so runner assignments +# cannot append them again. +RUNNER_MAKE := $(MAKE) --no-print-directory -f Makefile.runner \ + OUTPUT=$(OUTPUT) CC='$(CC)' CLANG='$(CLANG)' \ + CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' + +# Shared inputs prepared before runner sub-makes start. +# Also build veristat, which test_veristat executes. +RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ + $(TRUNNER_BPFTOOL) $(RESOLVE_BTFIDS) \ + $(OUTPUT)/veristat \ + $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ + $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ + prog_tests/tests.h map_tests/tests.h \ + $(RUNNER_OBJS) + +# Runtime fixtures for each test_progs flavor. +RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ + $(OUTPUT)/liburandom_read.so \ + $(OUTPUT)/xdp_synproxy \ + $(OUTPUT)/sign-file \ + $(OUTPUT)/uprobe_multi \ + $(TEST_KMOD_TARGETS) \ + ima_setup.sh \ + $(VERIFY_SIG_SETUP) \ + $(wildcard progs/btf_dump_test_case_*.c) \ + $(wildcard progs/*.bpf.o) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected + +# Prepare fixtures in parallel with runner compilation. +# Copy into flavor and out-of-tree directories; the in-tree +# default runner uses the originals. +RUNNER_EXTRA_DIRS := $(addprefix $(OUTPUT)/,$(TEST_INST_SUBDIRS)) +ifneq ($(realpath $(OUTPUT)),$(CURDIR)) +RUNNER_EXTRA_DIRS += $(OUTPUT) +test_progs: $(OUTPUT)/runner-extras +else +test_progs: $(RUNNER_EXTRA_FILES) +endif +RUNNER_EXTRAS := $(addsuffix /runner-extras,$(RUNNER_EXTRA_DIRS)) + +.PHONY: $(RUNNER_EXTRAS) +$(RUNNER_EXTRAS): $(RUNNER_EXTRA_FILES) + $(Q)mkdir -p $(@D) + $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) \ + $(RUNNER_EXTRA_FILES) $(@D)/ + +all: $(RUNNER_EXTRAS) +$(addprefix test_progs-,$(TEST_INST_SUBDIRS)): test_progs-%: \ + $(OUTPUT)/%/runner-extras + +# The main Makefile cannot tell whether $(OUTPUT)/test_progs is stale - +# only the runner sub-make knows its full dependency graph. FORCE makes +# the delegating rules below always run; their dependents still rebuild +# on mtime only. +FORCE: + +# Build default BPF outputs here for standalone consumers and install. +# Standalone consumers depend on individual skeletons; unflavored +# runners depend on the complete set before their sub-makes start. +RDIR := $(OUTPUT) +FLAVOR := +BINARY := test_progs +BPF_CC := $(CLANG) +BPF_CC_MSG := CLNG-BPF +BPF_SYS_INCLUDES := $(CLANG_SYS_INCLUDES) +BPF_CC_FLAGS := -O2 $(BPF_TARGET_ENDIAN) -mcpu=v3 +BPF_DEFINES := -DENABLE_ATOMICS_TESTS +include Makefile.skel + +DEFAULT_RUNNER_ARGS := RUNNER=test_progs FLAVOR= TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_CC)' BPF_CC_MSG=$(BPF_CC_MSG) \ + BPF_SYS_INCLUDES='$(BPF_SYS_INCLUDES)' \ + BPF_CC_FLAGS='$(BPF_CC_FLAGS)' BPF_DEFINES=$(BPF_DEFINES) +CLANG_RUNNER_ARGS := BPF_CC='$(CLANG)' BPF_CC_MSG=CLNG-BPF \ + BPF_SYS_INCLUDES='$(CLANG_SYS_INCLUDES)' + +$(OUTPUT)/test_progs: $(RUNNER_PREREQS) $(BPF_OBJS) $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) $(DEFAULT_RUNNER_ARGS) + +$(OUTPUT)/test_progs-no_alu32: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=no_alu32 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v2' -# Define test_progs-cpuv4 test runner. ifneq ($(CLANG_CPUV4),) -TRUNNER_BPF_BUILD_RULE := CLANG_CPUV4_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs,cpuv4)) +$(OUTPUT)/test_progs-cpuv4: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=cpuv4 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v4' \ + BPF_DEFINES=-DENABLE_ATOMICS_TESTS endif -# Define test_progs BPF-GCC-flavored test runner. ifneq ($(BPF_GCC),) -TRUNNER_BPF_BUILD_RULE := GCC_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(call get_sys_includes,gcc,) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,bpf_gcc)) +# Sys includes come from the *host* gcc, not $(BPF_GCC), as the rule +# this replaces had it. +GCC_SYS_INCLUDES := $(call get_sys_includes,gcc,) + +$(OUTPUT)/test_progs-bpf_gcc: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=bpf_gcc TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_GCC)' BPF_CC_MSG=GCC-BPF \ + BPF_SYS_INCLUDES='$(GCC_SYS_INCLUDES)' \ + BPF_CC_FLAGS='-DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2' endif -# Define test_maps test runner. -TRUNNER_TESTS_DIR := map_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_maps.c -TRUNNER_LIB_SOURCES := -TRUNNER_EXTRA_FILES := -TRUNNER_BPF_BUILD_RULE := $$(error no BPF objects should be built) -TRUNNER_BPF_CFLAGS := -$(eval $(call DEFINE_TEST_RUNNER,test_maps)) +# test_maps compiles map_tests/*.c against the default flavor's skeletons +$(OUTPUT)/test_maps: $(RUNNER_PREREQS) $(BPF_OBJS) \ + $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_maps FLAVOR= TESTS_DIR=map_tests + +TEST_GEN_FILES += $(BPF_OBJS) # Define test_verifier test runner. # It is much simpler than test_maps/test_progs and sufficiently different from diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index b7e4d7e678ac..d2a0c0031b87 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -2,8 +2,9 @@ # # Toolchain, path and flag definitions for the BPF selftests build, # kept apart from the rules so that more than one makefile can include -# them. Included after ../lib.mk - the note above the Makefile's include -# explains how the flags are ordered around it. +# them. Included by the Makefile after ../lib.mk and by Makefile.runner +# in its place - the note above the Makefile's include explains how the +# flags are ordered around it. TOOLSDIR := $(abspath ../../..) LIBDIR := $(TOOLSDIR)/lib @@ -148,3 +149,14 @@ ifneq ($(CLANG_HAS_ARENA_ASAN),) LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h CFLAGS += -DHAS_BPF_ARENA_ASAN endif + +RUNNER_OBJS-test_progs := $(addprefix $(OUTPUT)/, \ + test_progs.o cgroup_helpers.o trace_helpers.o \ + network_helpers.o testing_helpers.o btf_helpers.o \ + cap_helpers.o unpriv_helpers.o sysctl_helpers.o \ + netlink_helpers.o jit_disasm_helpers.o io_helpers.o \ + test_loader.o xsk.o disasm.o disasm_helpers.o \ + json_writer.o bpftool_helpers.o usdt_1.o usdt_2.o \ + find_bit.o) +RUNNER_OBJS-test_maps := $(addprefix $(OUTPUT)/, \ + test_maps.o testing_helpers.o) diff --git a/tools/testing/selftests/bpf/Makefile.runner b/tools/testing/selftests/bpf/Makefile.runner new file mode 100644 index 000000000000..d266c4246888 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.runner @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Build one test_progs flavor or test_maps. +# Invoked by the main Makefile after shared build inputs are ready. +# Flavored outputs live in $(OUTPUT)/$(FLAVOR); unflavored outputs +# share $(OUTPUT) with the main Makefile. +# +# Command-line parameters: +# OUTPUT selftests output directory, resolved by lib.mk +# RUNNER test_progs | test_maps +# FLAVOR empty | no_alu32 | cpuv4 | bpf_gcc +# TESTS_DIR prog_tests | map_tests +# BPF_* see Makefile.skel; BPF_CC is empty for test_maps +# CC, CLANG compilers resolved by the main Makefile +# CFLAGS, LDFLAGS assembled flags, including lib.mk's additions +# +# Exported by the main Makefile: +# LLVM_LDLIBS, LLVM_LDFLAGS: LLVM feature-probe results + +include ../../../build/Build.include +include ../../../scripts/Makefile.arch +include ../../../scripts/Makefile.include + +# Same message helpers as ../lib.mk, which only the top level includes. +ifeq ($(V),1) +Q = +msg = +else +Q = @ +msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))"; +MAKEFLAGS += --no-print-directory +endif + +include Makefile.buildvars + +BINARY := $(RUNNER)$(if $(FLAVOR),-$(FLAVOR)) +RDIR := $(OUTPUT)$(if $(FLAVOR),/$(FLAVOR)) + +all: $(OUTPUT)/$(BINARY) +.PHONY: all + +# Delete partially updated (corrupted) files on error +.DELETE_ON_ERROR: + +# No built-in rules: every file built here has an explicit rule, and the +# objects that come from the main Makefile have to be an error when +# missing rather than a silent rebuild by the built-in %.o recipes. +MAKEFLAGS += -r + +# Permissive build behaviour (skip-on-failure compile, partial-link) only +# applies to test_progs and its flavors; runners that use strong cross-object +# references (e.g. test_maps) keep strict semantics even when permissive. +PERMISSIVE_TESTS := $(if $(filter test_progs,$(RUNNER)),$(PERMISSIVE)) + +$(RDIR): + $(call msg,MKDIR,,$@) + $(Q)mkdir -p $@ + +# Per-runner sources + +ifeq ($(RUNNER),test_progs) +GENERATED_HDRS := $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) +endif + +TEST_SRCS := $(notdir $(wildcard $(TESTS_DIR)/*.c)) +TEST_OBJS := $(patsubst %.c,$(RDIR)/%.test.o,$(TEST_SRCS)) +TEST_DEPS := $(TEST_OBJS:.o=.d) + +# BPF objects and skeletons - rules shared with the main Makefile + +include Makefile.skel + +# When the compiler generates a %.d file, only skel basenames (not +# full paths) are specified as prerequisites for corresponding %.o +# file. vpath directives below instruct make to search for skel files +# in $(RDIR), if they are not present in the working directory. +vpath %.skel.h $(RDIR) +vpath %.lskel.h $(RDIR) +vpath %.subskel.h $(RDIR) + +# Test objects and runner binary + +$(RDIR)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(RDIR)/btf_data.bpf.o +$(RDIR)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 + +# compile individual test files +# Note: we cd into output directory to ensure embedded BPF object is found +# The %.test.d dependency files are a side effect of the -MMD below; the +# separate no-recipe rule for them orders the first compilation after +# skeleton generation and re-orders test object compilation when headers +# change. +$(TEST_OBJS): $(RDIR)/%.test.o: $(TESTS_DIR)/%.c | $(RDIR)/%.test.d + $(call msg,TEST-OBJ,$(BINARY),$@) + $(Q)(cd $(@D) && $(CC) -I. $(CFLAGS) -MMD -MT $@ -c $(CURDIR)/$< $(LDLIBS) -o $(@F)) $(if \ + $(PERMISSIVE_TESTS),$(call skip_on_fail,TEST)) + $(if $(TEST_NEEDS_BTFIDS), \ + $(Q)if [ -f $@ ]; then \ + $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(BINARY)]" "$(notdir $@)"); \ + $(RESOLVE_BTFIDS) --btf $(RDIR)/btf_data.bpf.o $@; \ + $(RESOLVE_BTFIDS) --patch_btfids $@.BTF_ids $@; \ + fi) + +$(TEST_DEPS): $(RDIR)/%.test.d: $(TESTS_DIR)/%.c $(GENERATED_HDRS) $(BPFOBJ) \ + | $(RDIR) $(ALL_SKELS) + +include $(wildcard $(TEST_DEPS)) + +cmd_runner_link = $(CC) $(CFLAGS) \ + $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)) \ + $(filter-out $(TEST_OBJS),$(filter %.a %.o,$^)), \ + $(filter %.a %.o,$^)) \ + $(LDLIBS) $(LLVM_LDLIBS) $(LDFLAGS) $(LLVM_LDFLAGS) -o $@ + +# BPF objects are also needed at runtime. +# In permissive mode, existing test objects are normal prerequisites +# so edits trigger relinking. Order-only prerequisites attempt all +# tests; the recipe links the objects that exist afterward. +$(OUTPUT)/$(BINARY): $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)),$(TEST_OBJS)) \ + $(RUNNER_OBJS-$(RUNNER)) $(BPFOBJ) \ + $(TRUNNER_BPFTOOL) \ + | $(BPF_OBJS) \ + $(if $(PERMISSIVE_TESTS),$(TEST_OBJS)) + $(call msg,BINARY,,$@) + $(Q)$(cmd_runner_link) + $(Q)ln -sf $(if $(FLAVOR),..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool $(RDIR)/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel new file mode 100644 index 000000000000..580d1d82c186 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0 +# BPF object and skeleton rules for one flavor. +# Included by the main Makefile for default outputs and by runners +# for their flavor. Unflavored runners require the default outputs first. +# +# Parameters (variables defined by the includer): +# RDIR output directory of this flavor +# FLAVOR flavor name, empty for the default +# BINARY runner binary name, for build-log messages +# BPF_CC BPF compiler +# BPF_CC_MSG build-log tag (CLNG-BPF/GCC-BPF) +# BPF_CC_FLAGS per-flavor compiler flags +# BPF_SYS_INCLUDES system include flags for $(BPF_CC) +# BPF_DEFINES extra defines (-DENABLE_ATOMICS_TESTS) + +# BPF objects and skeletons + +ifneq ($(BPF_CC),) + +BPF_SRCS := $(notdir $(wildcard progs/*.c)) +BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) + +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c + +LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ + linked_vars.skel.h linked_maps.skel.h \ + test_subskeleton.skel.h test_subskeleton_lib.skel.h \ + test_usdt.skel.h tracing_multi.skel.h \ + tracing_multi_module.skel.h \ + tracing_multi_intersect.skel.h \ + tracing_multi_session.skel.h + +LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ + core_kern.c core_kern_overflow.c test_ringbuf.c \ + test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ + test_ringbuf_overwrite.c + +LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c + +# Generate both light skeleton and libbpf skeleton for these +LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ + kfunc_call_test_subprog.c test_global_percpu_data.c +SKEL_BLACKLIST += $(LSKELS) $(LSKELS_SIGNED) + +test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o +linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o +linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o +linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o +# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file +# but that's created as a side-effect of the skel.h generation. +test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o +test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o +test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o +tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o +tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o + +LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) +LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) + +SKELS := $(patsubst %.c,$(RDIR)/%.skel.h, \ + $(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),$(BPF_SRCS))) +LSKELS_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS) $(LSKELS_EXTRA)) +LSKELS_SIGNED_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS_SIGNED)) +LINKED_SKELS_H := $(addprefix $(RDIR)/,$(LINKED_SKELS)) +ALL_SKELS := $(SKELS) $(LSKELS_H) $(LSKELS_SIGNED_H) $(LINKED_SKELS_H) + +HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ + $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ + $(addprefix $(BPFDIR)/, bpf_core_read.h \ + bpf_endian.h \ + bpf_helpers.h \ + bpf_tracing.h) + +# The following tests contain C code that, although technically legal, +# triggers GCC warnings that cannot be disabled: declaration of +# anonymous struct types in function parameter lists. +progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error + +cmd_bpf_compile = $(BPF_CC) $(BPF_CFLAGS) $(BPF_SYS_INCLUDES) $(BPF_DEFINES) \ + $($<-CFLAGS) $($<-$(FLAVOR)-CFLAGS) $(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF) + +$(BPF_OBJS): $(RDIR)/%.bpf.o: progs/%.c progs/*.h \ + $(INCLUDE_DIR)/vmlinux.h $(HEADERS_FOR_BPF_OBJS) \ + | $(RDIR) $(BPFOBJ) + $(call msg,$(BPF_CC_MSG),$(BINARY),$@) + $(Q)$(cmd_bpf_compile) + +GEN_SKEL := ./gen_bpf_skel.sh + +cmd_bpf_gen_skel = $(call skip_if_missing,SKEL,$<,$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) $< $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +$(SKELS): $(RDIR)/%.skel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_skel) + +cmd_bpf_gen_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $*_lskel --lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_lskel) + +cmd_bpf_gen_signed_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY) (signed)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) PRIVATE_KEY=$(PRIVATE_KEY) \ + VERIFICATION_CERT=$(VERIFICATION_CERT) \ + $(GEN_SKEL) --sign --name $*_lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_SIGNED_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) \ + $(GEN_SKEL) $(PRIVATE_KEY) $(VERIFICATION_CERT) | $(RDIR) + $(Q)$(cmd_bpf_gen_signed_lskel) + +cmd_bpf_link_skel = $(call skip_if_missing,SKEL,$(addprefix $(RDIR)/,\ + $($*.skel.h-deps)),$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'LINK-BPF' ' [$(BINARY)]' '$(notdir $(@:.skel.h=.bpf.o))'; \ + printf ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) \ + $(addprefix $(RDIR)/,$($*.skel.h-deps)) $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites +.SECONDEXPANSION: +$(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) \ + $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_link_skel) + +endif # BPF_CC From 34b1847924433ec088709d703bf3f93657e5a686 Mon Sep 17 00:00:00 2001 From: Fuyu Zhao Date: Tue, 15 Sep 2026 20:41:03 +0800 Subject: [PATCH 188/660] libbpf: Support selective kernel module BTF loading via bpf_object_open_opts Add btf_module_allowlist and btf_module_allowlist_cnt fields to bpf_object_open_opts to limit which kernel module BTFs libbpf is allowed to load. When the option is not specified, the existing behavior remains unchanged. An explicitly specified empty allowlist prevents libbpf from consulting any kernel module BTFs. The allowlist limits which kernel module BTFs libbpf will consult wherever module BTF might be needed. Suggested-by: Andrii Nakryiko Signed-off-by: Fuyu Zhao Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260915124104.77287-2-zhaofuyu@vivo.com --- tools/lib/bpf/libbpf.c | 70 ++++++++++++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h | 16 +++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 2f53afc985cf..cd1ea1bb53cb 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -751,6 +751,8 @@ struct bpf_object { bool btf_modules_loaded; size_t btf_module_cnt; size_t btf_module_cap; + char **btf_module_allowlist; + ssize_t btf_module_allowlist_cnt; /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ char *log_buf; @@ -5843,6 +5845,21 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, return 0; } +static bool is_btf_mod_allowed(const struct bpf_object *obj, const char *name) +{ + ssize_t i; + + if (obj->btf_module_allowlist_cnt < 0) + return true; + + for (i = 0; i < obj->btf_module_allowlist_cnt; i++) { + if (strcmp(obj->btf_module_allowlist[i], name) == 0) + return true; + } + + return false; +} + static int load_module_btfs(struct bpf_object *obj) { struct bpf_btf_info info; @@ -5865,6 +5882,9 @@ static int load_module_btfs(struct bpf_object *obj) if (!kernel_supports(obj, FEAT_MODULE_BTF)) return 0; + if (obj->btf_module_allowlist_cnt == 0) + return 0; + while (true) { err = bpf_btf_get_next_id(id, &id); if (err && errno == ENOENT) @@ -5907,6 +5927,11 @@ static int load_module_btfs(struct bpf_object *obj) continue; } + if (!is_btf_mod_allowed(obj, name)) { + close(fd); + continue; + } + btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { @@ -5931,6 +5956,9 @@ static int load_module_btfs(struct bpf_object *obj) break; } obj->btf_module_cnt++; + + if (obj->btf_module_allowlist_cnt == obj->btf_module_cnt) + break; } if (err) { @@ -8481,6 +8509,8 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, const struct bpf_object_open_opts *opts) { const char *kconfig, *btf_tmp_path, *token_path; + size_t mod_allow_cnt, i, j; + const char **mod_allow; struct bpf_object *obj; int err; char *log_buf; @@ -8525,6 +8555,22 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, if (token_path && strlen(token_path) >= PATH_MAX) return ERR_PTR(-ENAMETOOLONG); + mod_allow = OPTS_GET(opts, btf_module_allowlist, NULL); + mod_allow_cnt = OPTS_GET(opts, btf_module_allowlist_cnt, 0); + + if (mod_allow_cnt > SSIZE_MAX || (!mod_allow && mod_allow_cnt > 0)) + return ERR_PTR(-EINVAL); + + for (i = 0; i < mod_allow_cnt; i++) { + if (!mod_allow[i] || !mod_allow[i][0]) + return ERR_PTR(-EINVAL); + + for (j = 0; j < i; j++) { + if (strcmp(mod_allow[i], mod_allow[j]) == 0) + return ERR_PTR(-EINVAL); + } + } + obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); if (IS_ERR(obj)) return obj; @@ -8563,6 +8609,24 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, } } + obj->btf_module_allowlist_cnt = mod_allow ? mod_allow_cnt : -1; + if (mod_allow_cnt > 0) { + obj->btf_module_allowlist = + calloc(mod_allow_cnt, sizeof(*obj->btf_module_allowlist)); + if (!obj->btf_module_allowlist) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < mod_allow_cnt; i++) { + obj->btf_module_allowlist[i] = strdup(mod_allow[i]); + if (!obj->btf_module_allowlist[i]) { + err = -ENOMEM; + goto out; + } + } + } + err = bpf_object__elf_init(obj); err = err ? : bpf_object__elf_collect(obj); err = err ? : bpf_object__collect_externs(obj); @@ -9685,6 +9749,12 @@ void bpf_object__close(struct bpf_object *obj) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); + if (obj->btf_module_allowlist) { + for (i = 0; i < obj->btf_module_allowlist_cnt; i++) + zfree(&obj->btf_module_allowlist[i]); + zfree(&obj->btf_module_allowlist); + } + free(obj); } diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 026932f20962..03f794f4adb0 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -224,10 +224,24 @@ struct bpf_object_open_opts { * point (/sys/fs/bpf), in case this default behavior is undesirable. */ const char *bpf_token_path; + /* + * Optional allowlist of kernel module names whose BTFs libbpf is + * allowed to load. The allowlist limits which kernel module BTFs libbpf + * will consult wherever module BTF might be needed. + * + * When the option is not specified, the existing behavior remains + * unchanged. An explicitly specified empty list prevents libbpf from + * consulting any kernel module BTFs. + * + * The list must contain valid, non-empty module names and must not + * contain duplicate entries; otherwise -EINVAL is returned. + */ + const char **btf_module_allowlist; + size_t btf_module_allowlist_cnt; size_t :0; }; -#define bpf_object_open_opts__last_field bpf_token_path +#define bpf_object_open_opts__last_field btf_module_allowlist_cnt /** * @brief **bpf_object__open()** creates a bpf_object by opening From 66c22d8c0f62a6dfb4e9eaa58fddb804126f683d Mon Sep 17 00:00:00 2001 From: Fuyu Zhao Date: Tue, 15 Sep 2026 20:41:04 +0800 Subject: [PATCH 189/660] selftests/bpf: Add tests for selective module BTF loading Add selftests covering selective kernel module BTF loading through bpf_object_open_opts. The tests verify that: - the existing behavior is preserved when the allowlist is not specified; - loading succeeds when the required module BTF is specified in the allowlist; - module BTFs not in the allowlist are skipped; - an empty allowlist skips loading all module BTFs; - invalid allowlist and module name inputs are rejected. Signed-off-by: Fuyu Zhao Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260915124104.77287-3-zhaofuyu@vivo.com --- .../bpf/prog_tests/btf_module_allowlist.c | 162 ++++++++++++++++++ .../bpf/progs/btf_module_allowlist.c | 13 ++ 2 files changed, 175 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c create mode 100644 tools/testing/selftests/bpf/progs/btf_module_allowlist.c diff --git a/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c new file mode 100644 index 000000000000..3189133ee58b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "btf_module_allowlist.skel.h" + +static void test_load_with_list(const char **list, size_t count, + int expected_ret) +{ + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_EQ(ret, expected_ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_default(void) +{ + struct btf_module_allowlist *skel; + int ret; + + skel = btf_module_allowlist__open(); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_OK(ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_allow(void) +{ + const char *mod_list[] = { "bpf_testmod" }; + + test_load_with_list(mod_list, 1, 0); +} + +/* + * A non-NULL allowlist with a count of 0 explicitly disables module BTFs. + * The pointer must be non-NULL to distinguish this from an unspecified + * allowlist. + */ +static void btf_module_allowlist_emptylist(void) +{ + const char *mod_list[] = { NULL }; + + test_load_with_list(mod_list, 0, -ESRCH); +} + +/* + * bpf_testmod is not in the allowlist, so its BTF should not be loaded. + */ +static void btf_module_allowlist_skipunlisted(void) +{ + const char *mod_list[] = { "module_nonexist" }; + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = mod_list, + .btf_module_allowlist_cnt = 1, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = bpf_program__set_attach_target(skel->progs.test_btf_module_allowlist, 0, + "bpf_testmod:bpf_testmod_fentry_test1"); + ASSERT_EQ(ret, -ESRCH, "bpf_program__set_attach_target"); + btf_module_allowlist__destroy(skel); +} + +static void test_invalid_input(const char **list, size_t count, + const char *test_name) +{ + struct btf_module_allowlist *skel; + char assert_name[64]; + int err; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + snprintf(assert_name, sizeof(assert_name), "%s: open_opts", test_name); + skel = btf_module_allowlist__open_opts(&opts); + err = errno; + if (!ASSERT_NULL(skel, assert_name)) { + btf_module_allowlist__destroy(skel); + return; + } + snprintf(assert_name, sizeof(assert_name), "%s: open_opts err", test_name); + ASSERT_EQ(err, EINVAL, assert_name); +} + +static void btf_module_allowlist_invalidinput(void) +{ + const char *names[] = { NULL }; + const char *empty_names[] = { "" }; + const char *duplicate_names[] = { "bpf_testmod", "bpf_testmod" }; + + test_invalid_input(NULL, 1, "null_list"); + test_invalid_input(names, 1, "null_name"); + test_invalid_input(empty_names, 1, "empty_name"); + test_invalid_input(duplicate_names, 2, "duplicate_name"); + test_invalid_input(names, (size_t)SSIZE_MAX + 1, "large_count"); +} + +void test_btf_module_allowlist(void) +{ + struct btf *vmlinux_btf = NULL; + struct btf *module_btf = NULL; + + if (!env.has_testmod) { + printf("%s:SKIP: bpf_testmod is not available\n", __func__); + test__skip(); + return; + } + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) { + printf("%s:SKIP: vmlinux_btf is not available\n", __func__); + test__skip(); + return; + } + + /* Ensure bpf_testmod BTF is available. */ + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + printf("%s:SKIP: bpf_testmod's BTF is not available\n", __func__); + btf__free(vmlinux_btf); + test__skip(); + return; + } + + btf__free(module_btf); + btf__free(vmlinux_btf); + + if (test__start_subtest("btf_module_allowlist_default")) + btf_module_allowlist_default(); + + if (test__start_subtest("btf_module_allowlist_allow")) + btf_module_allowlist_allow(); + + if (test__start_subtest("btf_module_allowlist_skipunlisted")) + btf_module_allowlist_skipunlisted(); + + if (test__start_subtest("btf_module_allowlist_emptylist")) + btf_module_allowlist_emptylist(); + + if (test__start_subtest("btf_module_allowlist_invalidinput")) + btf_module_allowlist_invalidinput(); +} diff --git a/tools/testing/selftests/bpf/progs/btf_module_allowlist.c b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c new file mode 100644 index 000000000000..bb3c38bee77d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +SEC("fentry/bpf_testmod_loop_test") +int BPF_PROG(test_btf_module_allowlist) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; From 06046a8a50e85165cfe4cf1b99ac4c0d43781189 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:25 +0200 Subject: [PATCH 190/660] selftests/bpf: Allow privileged preparation for capability tests The annotation-driven loader drops capabilities before libbpf prepares an object. Resolving bpf_testmod kfuncs requires CAP_SYS_ADMIN to enumerate and open module BTF, so tests without that capability fail before reaching the verifier. Add an opt-in __prepare_priv annotation. Call bpf_object__prepare() with the fixture's initial capabilities, then apply __caps_unpriv before loading the programs. This uses libbpf's explicit prepare/load boundary. In particular, CAP_SYS_ADMIN must be dropped along with CAP_PERFMON to test uninitialized stack checks, since CAP_SYS_ADMIN satisfies the verifier's CAP_PERFMON check. Preparation also creates maps and loads BTF. Keep it opt-in so existing tests continue checking those operations with reduced capabilities. The existing pre-execution callback runs after program loading and is too late for this. Allow tests retaining CAP_BPF to run when the unprivileged-BPF sysctl is set. Check CPU mitigations separately: disabled or undetectable mitigations must still skip these tests, because CAP_BPF does not restore speculative execution checks. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-2-memxor@gmail.com --- tools/testing/selftests/bpf/progs/bpf_misc.h | 9 +++- tools/testing/selftests/bpf/test_loader.c | 48 ++++++++++++++------ tools/testing/selftests/bpf/unpriv_helpers.c | 16 +++++-- tools/testing/selftests/bpf/unpriv_helpers.h | 2 + 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index eb88d9ce6c34..2ced1d751ace 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -9,7 +9,8 @@ #define QUOTE(str) #str #define EXPAND_QUOTE(str) QUOTE(str) -/* This set of attributes controls behavior of the +/* + * This set of attributes controls behavior of the * test_loader.c:test_loader__run_subtests(). * * The test_loader sequentially loads each program in a skeleton. @@ -131,6 +132,11 @@ * Several __arch_* annotations could be specified at once. * When test case is not run on current arch it is marked as skipped. * __caps_unpriv Specify the capabilities that should be set when running the test. + * __prepare_priv In unprivileged mode, prepare the object with the fixture's + * initial capabilities before dropping them for program loading. + * Preparation includes map creation and BTF/kfunc resolution; + * these operations are not tested at the reduced capabilities. + * Program loading uses the normal __caps_unpriv selection. * * __linear_size Specify the size of the linear area of non-linear skbs, or * 0 for linear skbs. @@ -166,6 +172,7 @@ #define __arch_s390x __arch("s390x") #define __arch_loongarch __arch("LOONGARCH") #define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) +#define __prepare_priv __test_tag("test_prepare_priv") #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") #define __stderr(msg) __test_tag("test_expect_stderr=" msg) diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 28724de06322..a6e3fcc1079c 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -33,6 +33,7 @@ static inline const char *str_has_pfx(const char *str, const char *pfx) #endif static int sysctl_unpriv_disabled = -1; +static int unpriv_mitigations_disabled = -1; enum mode { PRIV = 1, @@ -71,6 +72,7 @@ struct test_spec { int load_mask; int linear_sz; const char *skip_reason; + bool prepare_priv; bool auxiliary; bool valid; }; @@ -606,6 +608,8 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if (strcmp(s, "test_prepare_priv") == 0) { + spec->prepare_priv = true; } else if ((val = str_has_pfx(s, "load_mode="))) { if (strcmp(val, "jited") == 0) { load_mask = JITED; @@ -1015,10 +1019,10 @@ struct cap_state { bool initialized; }; -static int drop_capabilities(struct cap_state *caps) +static int drop_capabilities(struct cap_state *caps, __u64 keep_caps) { const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | - 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); + 1ULL << CAP_PERFMON | 1ULL << CAP_BPF) & ~keep_caps; int err; err = cap_disable_effective(caps_to_drop, &caps->old_caps); @@ -1028,6 +1032,13 @@ static int drop_capabilities(struct cap_state *caps) } caps->initialized = true; + if (keep_caps) { + err = cap_enable_effective(keep_caps, NULL); + if (err) { + PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); + return err; + } + } return 0; } @@ -1048,8 +1059,12 @@ static int restore_capabilities(struct cap_state *caps) static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) { if (sysctl_unpriv_disabled < 0) - sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; - if (sysctl_unpriv_disabled) + sysctl_unpriv_disabled = get_unpriv_sysctl_disabled(); + if (sysctl_unpriv_disabled && !(spec->unpriv.caps & (1ULL << CAP_BPF))) + return false; + if (unpriv_mitigations_disabled < 0) + unpriv_mitigations_disabled = get_unpriv_mitigations_disabled(); + if (unpriv_mitigations_disabled) return false; if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) return false; @@ -1351,17 +1366,8 @@ void run_subtest(struct test_loader *tester, test__end_subtest(); return; } - if (drop_capabilities(&caps)) { - test__end_subtest(); - return; - } - if (subspec->caps) { - err = cap_enable_effective(subspec->caps, NULL); - if (err) { - PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); - goto subtest_cleanup; - } - } + if (!spec->prepare_priv && drop_capabilities(&caps, subspec->caps)) + goto subtest_cleanup; } /* Implicitly reset to NULL if next test case doesn't specify. @@ -1414,6 +1420,18 @@ void run_subtest(struct test_loader *tester, bpf_object__for_each_map(map, tobj) bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); + if (unpriv && spec->prepare_priv) { + /* + * Module BTF lookup needs CAP_SYS_ADMIN. Allow tests to prepare + * their objects first, then verify programs with the requested caps. + */ + err = bpf_object__prepare(tobj); + if (!ASSERT_OK(err, "obj_prepare")) + goto tobj_cleanup; + if (drop_capabilities(&caps, subspec->caps)) + goto tobj_cleanup; + } + err = bpf_object__load(tobj); if (subspec->expect_failure) { if (!ASSERT_ERR(err, "unexpected_load_success")) { diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index 2c8c5edb8751..c8dd5d848584 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -111,9 +111,8 @@ static int get_mitigations_off(void) return !enabled_in_config; } -bool get_unpriv_disabled(void) +bool get_unpriv_sysctl_disabled(void) { - int mitigations_off; bool disabled; char buf[2]; FILE *fd; @@ -127,8 +126,12 @@ bool get_unpriv_disabled(void) disabled = true; } - if (disabled) - return true; + return disabled; +} + +bool get_unpriv_mitigations_disabled(void) +{ + int mitigations_off; /* * Some unpriv tests rely on spectre mitigations being on. @@ -144,6 +147,11 @@ bool get_unpriv_disabled(void) return mitigations_off; } +bool get_unpriv_disabled(void) +{ + return get_unpriv_sysctl_disabled() || get_unpriv_mitigations_disabled(); +} + bool get_kasan_jit_enabled(void) { return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index a7ceb51577cd..c24d53e14f3a 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h @@ -5,5 +5,7 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_unpriv_sysctl_disabled(void); +bool get_unpriv_mitigations_disabled(void); bool get_kasan_jit_enabled(void); bool get_kasan_multi_shot_enabled(void); From e2b4aaa75040dd68f97dce8a7218e088243666b0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 21 Sep 2026 04:38:26 +0200 Subject: [PATCH 191/660] bpf: Record raw memory arguments during argument checking Record raw outputs in check_func_arg() after resolving their memory type, instead of pre-recording them in check_raw_mode_ok(). Keep prototype validation separate from per-call output tracking so kfuncs can share the latter in a following patch. For bpf_strtol(), bpf_strtoul() and bpf_kallsyms_lookup_name(), a variable-size input is checked before the arg4 output. Recording outputs as their arguments are checked prevents that input from clearing the output's raw mode early. Only clear raw mode when a variable size belongs to the recorded output: an unrelated later input must also preserve an earlier output descriptor. For bpf_map_peek_elem() on a bloom filter, type resolution removes MEM_UNINIT and MEM_WRITE because the value is an input. Recording the resolved type makes the later bloom-filter-specific raw-mode reset unnecessary. Reuse the same raw-memory predicate for output recording and prototype validation. Signed-off-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-3-memxor@gmail.com --- kernel/bpf/verifier.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 18aad4886f9c..67c1abfde922 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7217,12 +7217,13 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, */ meta->msize_max_value = reg_umax(size_reg); - /* The register is SCALAR_VALUE; the access check happens using - * its boundaries. For unprivileged variable accesses, disable - * raw mode so that the program is required to initialize all - * the memory that the helper could just partially fill up. + /* + * A variable size does not guarantee that the call initializes the whole + * checked range. Disable raw mode for this output and apply the ordinary + * stack initialization checks, including their privilege exceptions. */ - if (!tnum_is_const(size_reg->var_off)) + if (!tnum_is_const(size_reg->var_off) && + meta->arg_raw_mem.regno == reg_from_argno(mem_argno)) meta->arg_raw_mem.regno = 0; if (reg_smin(size_reg) < 0) { @@ -8940,6 +8941,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err) return err; + if (!meta->btf && arg_type_is_raw_mem(arg_type)) + meta->arg_raw_mem.regno = slot + 1; + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { err = mark_arg_precision(env, argno); if (err) @@ -9029,14 +9033,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p return -EFAULT; } - /* - * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads - * the value buffer as an input rather than filling it. - */ - if (is_helper_call(meta, BPF_FUNC_map_peek_elem) && - meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) - meta->arg_raw_mem.regno = 0; - err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); @@ -9860,8 +9856,9 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +static bool check_raw_mode_ok(const struct bpf_func_proto *fn) { + bool seen = false; int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { @@ -9869,9 +9866,9 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a break; if (!arg_type_is_raw_mem(fn->arg_type[i])) continue; - if (meta->arg_raw_mem.regno) + if (seen) return false; - meta->arg_raw_mem.regno = i + 1; + seen = true; } return true; @@ -10003,7 +10000,7 @@ static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_ struct bpf_call_arg_meta *meta) { return check_arg_prog_aux(env, fn) && - check_raw_mode_ok(fn, meta) && + check_raw_mode_ok(fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && From fc8dc2be101b038adf6c36c51511a53ca2407199 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:27 +0200 Subject: [PATCH 192/660] bpf: Check __uninit kfunc output buffers as write-only Generic kfunc memory arguments are checked for both read and write access, including __uninit outputs. An output pointing into a write-only map value is therefore rejected, and a stack output must already hold readable contents even though the kfunc only writes it. Mark generic buffers in generated kfunc prototypes with MEM_WRITE, and keep MEM_UNINIT when a struct pointer resolves to generic memory. Check __uninit buffers as write-only while ordinary kfunc buffers retain read/write access. Helper access selection is unchanged. Definite initialization of stack outputs after the call is addressed separately. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-4-memxor@gmail.com --- kernel/bpf/verifier.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 67c1abfde922..de40785d6a7b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9190,7 +9190,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p break; access_type = arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ; - if (meta->btf) + /* Ordinary kfunc buffers are input/output; __uninit buffers are outputs. */ + if (meta->btf && !(arg_type & MEM_UNINIT)) access_type = BPF_READ | BPF_WRITE; err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); @@ -9238,7 +9239,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p break; access_type = fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ; - if (meta->btf) + if (meta->btf && !(fn->arg_type[arg - 1] & MEM_UNINIT)) access_type = BPF_READ | BPF_WRITE; zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; @@ -12457,7 +12458,8 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, PTR_ERR(resolve_ret)); return -EINVAL; } - *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | (*arg_type & PTR_MAYBE_NULL); + *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE | + (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT)); return 0; } @@ -12970,7 +12972,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = ARG_PTR_TO_MEM; + arg_type = ARG_PTR_TO_MEM | MEM_WRITE; } else if (btf_type_is_struct(ref_t)) /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ arg_type = ARG_PTR_TO_BTF_ID; @@ -12995,7 +12997,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, return -EINVAL; } proto->arg_size[arg] = type_size; - arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE; } if (is_kfunc_arg_uninit(meta->btf, &args[arg])) From fc670d4b6c31d69f5e872a397d17fe4c5f970ffe Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:28 +0200 Subject: [PATCH 193/660] bpf: Fix generic __uninit kfunc output buffers Stack liveness treats __uninit kfunc arguments as writes. Even with write-only access checks, ordinary clobber handling leaves poisoned stack bytes poisoned after the call, so the output cannot be read. Reuse the helper output descriptor for generic kfunc buffers. Record the output after resolving its memory type, and initialize its stack bytes only after all arguments have been checked. This prevents an output from making an aliased, uninitialized input appear valid. Track the output by its ABI argument slot, since a kfunc pointer may be passed on the stack or follow a multi-slot by-value argument. Keep the single-output restriction in prototype validation; multiple-output tracking is a separate extension. Document that output buffers must be fully initialized on every return path, including error returns and padding. Fixes: 2cb27158adb3 ("bpf: poison dead stack slots") Reported-by: Tejun Heo Suggested-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/86d966ec88bbf27d21b2bb4e18c8aa00@kernel.org Link: https://patch.msgid.link/20260921023843.411943-5-memxor@gmail.com --- Documentation/bpf/kfuncs.rst | 26 ++++++++---- include/linux/bpf_verifier.h | 7 ++-- kernel/bpf/verifier.c | 80 ++++++++++++++++++++++++++---------- 3 files changed, 81 insertions(+), 32 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 3f300118a623..f393c3c3d3b4 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -164,19 +164,31 @@ suffix should be used. 2.3.3 __uninit Annotation ------------------------- -This annotation is used to indicate that the argument will be treated as -uninitialized. +Use ``__uninit`` on a pointer parameter for an output that the kfunc +initializes without reading its incoming contents. -An example is given below:: +For generic memory buffers, the kfunc must initialize every byte in the +declared range on every return path, including error returns and struct +padding. The range is determined by the pointed-to type or the associated +``__sz`` or ``__szk`` size argument. - __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr_kern *ptr__uninit) +The annotation does not change the accepted pointer types. A stack-backed +struct passed as a generic memory buffer must still be scalar-only. + +A stack buffer with a verifier-known constant offset and size may be +uninitialized before the call and is considered initialized afterwards. +For variable offsets or sizes, the usual stack-initialization and +variable-offset restrictions still apply. + +For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a +dynptr in the supplied storage. For example:: + + __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr *ptr__uninit) { ... } -Here, the dynptr will be treated as an uninitialized dynptr. Without this -annotation, the verifier will reject the program if the dynptr passed in is -not initialized. +Without this annotation, a dynptr argument must already be initialized. 2.3.4 __nullable Annotation --------------------------- diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index cf85141ea167..6ff1c227d298 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1548,10 +1548,11 @@ struct ref_obj_desc { /* * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access() + * after all arguments have been checked. */ struct arg_raw_mem_desc { - u8 regno; + u8 argno; /* One-based ABI argument slot; zero means no output. */ int size; }; @@ -1579,6 +1580,7 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct ret_mem_desc ret_mem; + struct arg_raw_mem_desc arg_raw_mem; /* Only set by kfunc */ bool r0_rdonly; @@ -1617,7 +1619,6 @@ struct bpf_call_arg_meta { s64 const_map_key; struct btf *ret_btf; struct btf_field *kptr_field; - struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index de40785d6a7b..f4b88e402ff5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -302,6 +302,12 @@ static int arg_idx_from_argno(argno_t a) return arg_from_argno(a) - 1; } +/* Normalize helper register numbers and kfunc argument numbers to ABI slots. */ +static u32 arg_slot_from_argno(argno_t a) +{ + return abs(a.argno) - 1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -6988,7 +6994,7 @@ static int check_stack_range_initialized( */ bool allow_poison = access_size < 0 || clobber; /* The call will initialize the memory; uninitialized stack allowed */ - bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); + bool raw_mode = meta && meta->arg_raw_mem.argno == arg_slot_from_argno(argno) + 1; access_size = abs(access_size); @@ -7017,11 +7023,9 @@ static int check_stack_range_initialized( reg_arg_name(env, argno), tn_buf); return -EACCES; } - /* Only initialized buffer on stack is allowed to be accessed - * with variable offset. With uninitialized buffer it's hard to - * guarantee that whole memory is marked as initialized on - * helper return since specific bounds are unknown what may - * cause uninitialized stack leaking. + /* + * The call may touch any byte in the possible range, but does not + * definitely initialize all of it. Fall back to ordinary stack checks. */ raw_mode = false; @@ -7223,8 +7227,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, * stack initialization checks, including their privilege exceptions. */ if (!tnum_is_const(size_reg->var_off) && - meta->arg_raw_mem.regno == reg_from_argno(mem_argno)) - meta->arg_raw_mem.regno = 0; + meta->arg_raw_mem.argno == arg_slot_from_argno(mem_argno) + 1) + meta->arg_raw_mem.argno = 0; if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -8941,8 +8945,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err) return err; - if (!meta->btf && arg_type_is_raw_mem(arg_type)) - meta->arg_raw_mem.regno = slot + 1; + if (arg_type_is_raw_mem(arg_type)) + meta->arg_raw_mem.argno = slot + 1; if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { err = mark_arg_precision(env, argno); @@ -9565,6 +9569,33 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met return 0; } +static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + u32 slot = meta->arg_raw_mem.argno - 1; + int i, err; + + if (!meta->arg_raw_mem.size) + return 0; + reg = get_func_arg_reg(caller, cur_regs(env), slot); + + /* + * Validate every argument before initializing outputs: an input argument + * may alias an output buffer. Use the normal stack-write checks to discard + * stale spills and preserve the rules for special stack objects. + */ + for (i = 0; i < meta->arg_raw_mem.size; i++) { + err = check_mem_access(env, insn_idx, reg, argno_from_arg(slot + 1), i, BPF_B, + BPF_WRITE, -1, false, false); + if (err) + return err; + } + + return 0; +} + static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; @@ -9863,9 +9894,13 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_UNUSED) + enum bpf_arg_type type = fn->arg_type[i]; + + if (type == ARG_UNUSED) break; - if (!arg_type_is_raw_mem(fn->arg_type[i])) + /* Struct pointers may resolve to generic memory during argument checking. */ + if (!arg_type_is_raw_mem(type) && + !(base_type(type) == ARG_PTR_TO_BTF_ID && (type & MEM_UNINIT))) continue; if (seen) return false; @@ -11588,16 +11623,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs = cur_regs(env); - /* Mark slots with STACK_MISC in case of raw mode, stack offset - * is inferred from register state. - */ - for (i = 0; i < meta.arg_raw_mem.size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, - argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, - BPF_WRITE, -1, false, false); - if (err) - return err; - } + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; @@ -13101,6 +13129,10 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg return -ENOTSUPP; } + if (!check_raw_mode_ok(proto)) { + verbose(env, "multiple __uninit buffers are not supported\n"); + return -EINVAL; + } return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } @@ -14178,6 +14210,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err < 0) return err; + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; + if ((is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ From a8bbd9ee1bbc73ae51508ca045bbb693bbdda446 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:29 +0200 Subject: [PATCH 194/660] selftests/bpf: Cover generic __uninit output initialization Exercise the struct and sized-buffer cases where stack liveness poisons an output before a kfunc call. Check that the verifier accepts these outputs and that the kfunc initializes the memory read after the call. Verify that an uninitialized input aliasing an output is still rejected without CAP_PERFMON or CAP_SYS_ADMIN. Include an initialized alias as a positive control, using an int-width store so its value is independent of endianness. Use __prepare_priv to resolve the test module's BTF before dropping to CAP_BPF and CAP_NET_ADMIN for program loading. Keep multiple-output and argument-slot coverage separate from these immediate regression tests. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Link: https://patch.msgid.link/20260921023843.411943-6-memxor@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_kfunc_uninit.c | 100 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 25 +++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 3 + 4 files changed, 130 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 973bbeda9318..ffeba2d1464a 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -56,6 +56,7 @@ #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" #include "verifier_kfunc_packet_access.skel.h" +#include "verifier_kfunc_uninit.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" @@ -223,6 +224,7 @@ void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } void test_verifier_kfunc_packet_access(void) { RUN_TESTS(verifier_kfunc_packet_access); } +void test_verifier_kfunc_uninit(void) { RUN_TESTS(verifier_kfunc_uninit); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c new file mode 100644 index 000000000000..f7818303e703 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_struct), + "r"(&bpf_kfunc_test_uninit_mem), + "r"(&bpf_kfunc_test_uninit_alias)); +} + +SEC("tc") +__success __retval(10) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void struct_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 12);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 8);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 4);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0x2a2a2a2a) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void sized_buffer_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u32 *)(r10 - 8);" + "r1 = *(u32 *)(r10 - 4);" + "if r0 == r1 goto +1;" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(7) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void initialized_input_alias(void) +{ + asm volatile ( + "*(u32 *)(r10 - 8) = 7;" + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +SEC("tc") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack") +__naked void uninitialized_input_alias(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index fd2c0cdc91b1..542edeb28b27 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1316,6 +1317,27 @@ __bpf_kfunc void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) { } +__bpf_kfunc void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) +{ + out__uninit->x0 = 1; + out__uninit->x1 = 2; + out__uninit->x2 = 3; + out__uninit->x3 = 4; +} + +__bpf_kfunc void bpf_kfunc_test_uninit_mem(void *out__uninit, u32 out__sz) +{ + memset(out__uninit, 0x2a, out__sz); +} + +__bpf_kfunc int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) +{ + int value = get_unaligned(in); + + put_unaligned(42, out__uninit); + return value; +} + __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) { } @@ -1747,6 +1769,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_struct) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_mem) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_alias) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index d3696d5254c9..91b64e783123 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -289,6 +289,9 @@ __u64 bpf_kfunc_call_stack_arg_big(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) __ksym; void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; +void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) __ksym; +void bpf_kfunc_test_uninit_mem(void *out__uninit, __u32 out__sz) __ksym; +int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) __ksym; void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym; void bpf_kfunc_call_test_destructive(void) __ksym; From bca730ba6df6d62fa7f94011fe089e666ccb65ad Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:30 +0200 Subject: [PATCH 195/660] bpf: Support multiple __uninit kfunc output arguments A single output descriptor cannot retain the initialization ranges of two __uninit memory arguments. Track raw-mode eligibility and the definite output size separately for each ABI slot, so a variable-size output leaves independent constant-size outputs intact. Use the shared argument-checking path to record outputs for both helpers and kfuncs. With per-slot tracking, neither needs the single-output prototype restriction. Initialize every recorded output after checking all arguments, skipping empty slots before looking up their register state. Use the resolved BTF parameter when looking up __uninit for stack liveness. A preceding by-value parameter can consume multiple ABI slots, so its slot number cannot index the BTF parameter array. Suggested-by: Amery Hung Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-7-memxor@gmail.com --- include/linux/bpf_verifier.h | 10 +++--- kernel/bpf/verifier.c | 68 ++++++++++++------------------------ 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6ff1c227d298..9ddbb20ec1e9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1547,13 +1547,13 @@ struct ref_obj_desc { }; /* - * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access() - * after all arguments have been checked. + * Memory arguments a call fills in, indexed by argument slot. The verifier allows the + * stack to be uninitialized if the range is a known constant. Stack slots are marked as + * STACK_MISC by check_mem_access() after all arguments have been checked. */ struct arg_raw_mem_desc { - u8 argno; /* One-based ABI argument slot; zero means no output. */ - int size; + u16 mask; + int size[MAX_BPF_FUNC_ARGS]; }; /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f4b88e402ff5..0c94f1214cc3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6994,7 +6994,9 @@ static int check_stack_range_initialized( */ bool allow_poison = access_size < 0 || clobber; /* The call will initialize the memory; uninitialized stack allowed */ - bool raw_mode = meta && meta->arg_raw_mem.argno == arg_slot_from_argno(argno) + 1; + u32 arg_slot = arg_slot_from_argno(argno); + bool raw_mode = meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); access_size = abs(access_size); @@ -7034,7 +7036,7 @@ static int check_stack_range_initialized( } if (raw_mode) { - meta->arg_raw_mem.size = access_size; + meta->arg_raw_mem.size[arg_slot] = access_size; return 0; } @@ -7226,9 +7228,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, * checked range. Disable raw mode for this output and apply the ordinary * stack initialization checks, including their privilege exceptions. */ - if (!tnum_is_const(size_reg->var_off) && - meta->arg_raw_mem.argno == arg_slot_from_argno(mem_argno) + 1) - meta->arg_raw_mem.argno = 0; + if (!tnum_is_const(size_reg->var_off)) + meta->arg_raw_mem.mask &= ~BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -8946,7 +8947,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p return err; if (arg_type_is_raw_mem(arg_type)) - meta->arg_raw_mem.argno = slot + 1; + meta->arg_raw_mem.mask |= BIT(slot); if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { err = mark_arg_precision(env, argno); @@ -9573,24 +9574,28 @@ static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta int insn_idx) { struct bpf_func_state *caller = cur_func(env); - struct bpf_reg_state *reg; - u32 slot = meta->arg_raw_mem.argno - 1; + u32 slot; int i, err; - if (!meta->arg_raw_mem.size) - return 0; - reg = get_func_arg_reg(caller, cur_regs(env), slot); - /* * Validate every argument before initializing outputs: an input argument * may alias an output buffer. Use the normal stack-write checks to discard * stale spills and preserve the rules for special stack objects. */ - for (i = 0; i < meta->arg_raw_mem.size; i++) { - err = check_mem_access(env, insn_idx, reg, argno_from_arg(slot + 1), i, BPF_B, - BPF_WRITE, -1, false, false); - if (err) - return err; + for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { + struct bpf_reg_state *reg; + argno_t argno = argno_from_arg(slot + 1); + + if (!meta->arg_raw_mem.size[slot]) + continue; + reg = get_func_arg_reg(caller, cur_regs(env), slot); + + for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B, + BPF_WRITE, -1, false, false); + if (err) + return err; + } } return 0; @@ -9888,28 +9893,6 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn) -{ - bool seen = false; - int i; - - for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - enum bpf_arg_type type = fn->arg_type[i]; - - if (type == ARG_UNUSED) - break; - /* Struct pointers may resolve to generic memory during argument checking. */ - if (!arg_type_is_raw_mem(type) && - !(base_type(type) == ARG_PTR_TO_BTF_ID && (type & MEM_UNINIT))) - continue; - if (seen) - return false; - seen = true; - } - - return true; -} - static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) { bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; @@ -10036,7 +10019,6 @@ static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_ struct bpf_call_arg_meta *meta) { return check_arg_prog_aux(env, fn) && - check_raw_mode_ok(fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -13129,10 +13111,6 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg return -ENOTSUPP; } - if (!check_raw_mode_ok(proto)) { - verbose(env, "multiple __uninit buffers are not supported\n"); - return -EINVAL; - } return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } @@ -13929,7 +13907,7 @@ out: /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) return -size; - if (is_kfunc_arg_uninit(btf, &args[arg])) + if (is_kfunc_arg_uninit(btf, &args[i])) return -size; return size; } From 89835699ef4e0240758429e0e85d3bbd4154065e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:31 +0200 Subject: [PATCH 196/660] selftests/bpf: Cover __uninit kfunc output argument slots Keep coverage for per-slot output tracking separate from the immediate single-output regression tests. Check that both constant-size outputs are initialized, and that a variable-size output does not disable initialization of an independent constant-size output. Also exercise an output following a by-value parameter that occupies two argument slots, and an output pointer passed on the stack. Run each case with normal capabilities and with CAP_BPF and CAP_NET_ADMIN only. Leave the stack-passed output uninitialized so its reduced-capability case fails if the verifier treats it as an ordinary input buffer. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-8-memxor@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_kfunc_uninit_multi.c | 113 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 20 ++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 4 + 4 files changed, 139 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index ffeba2d1464a..4f1e1c1cd5ab 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -57,6 +57,7 @@ #include "verifier_jit_convergence.skel.h" #include "verifier_kfunc_packet_access.skel.h" #include "verifier_kfunc_uninit.skel.h" +#include "verifier_kfunc_uninit_multi.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" @@ -225,6 +226,7 @@ void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null) void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } void test_verifier_kfunc_packet_access(void) { RUN_TESTS(verifier_kfunc_packet_access); } void test_verifier_kfunc_uninit(void) { RUN_TESTS(verifier_kfunc_uninit); } +void test_verifier_kfunc_uninit_multi(void) { RUN_TESTS(verifier_kfunc_uninit_multi); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c new file mode 100644 index 000000000000..18ced0d0a3b2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_multi), + "r"(&bpf_kfunc_test_uninit_pair), + "r"(&bpf_kfunc_test_uninit_stack)); +} + +SEC("tc") +__success __retval(42) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void multiple_outputs(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 8);" + "if r1 != 0x2a2a2a2a goto 1f;" + "r1 = *(u32 *)(r10 - 4);" + "if r1 == 0x2a2a2a2a goto 2f;" + "1:;" + "r0 = 0;" + "2:;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void variable_size_preserves_other_output(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 7;" + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void output_after_by_value_argument(void) +{ + asm volatile ( + "r1 = 20;" + "r2 = 22;" + "r3 = r10;" + "r3 += -8;" + "call %[bpf_kfunc_test_uninit_pair];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_pair) : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("tc") +__arch_x86_64 __arch_arm64 __arch_riscv64 +__success __retval(15) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void output_passed_on_stack(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "r6 = r10;" + "r6 += -8;" + "*(u64 *)(r11 - 8) = r6;" + "call %[bpf_kfunc_test_uninit_stack];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_stack) : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 542edeb28b27..211886a8ee87 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1338,6 +1338,23 @@ __bpf_kfunc int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) return value; } +__bpf_kfunc void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, u32 b__sz) +{ + put_unaligned(42, a__uninit); + memset(b__uninit, 0x2a, b__sz); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) +{ + put_unaligned((int)(p.lo + p.hi), out__uninit); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_stack(u64 a, u64 b, u64 c, u64 d, u64 e, + int *out__uninit) +{ + put_unaligned((int)(a + b + c + d + e), out__uninit); +} + __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) { } @@ -1772,6 +1789,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_struct) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_mem) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_alias) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_multi) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_pair) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_stack) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 91b64e783123..524f2cb9bdf4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -292,6 +292,10 @@ void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) __ksym; void bpf_kfunc_test_uninit_mem(void *out__uninit, __u32 out__sz) __ksym; int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) __ksym; +void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, __u32 b__sz) __ksym; +void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) __ksym; +void bpf_kfunc_test_uninit_stack(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + int *out__uninit) __ksym; void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym; void bpf_kfunc_call_test_destructive(void) __ksym; From 5da4a9f26fca03ccd8afcc88f5c709c459c00ab8 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:32 +0200 Subject: [PATCH 197/660] bpf: Preserve stack initialization for generic output buffers Partial-output helpers such as bpf_snprintf() do not read incoming buffer contents, but may leave some bytes untouched. Let them accept uninitialized storage without promising full initialization to callers that cannot read uninitialized stack memory. Make this the default for generic MEM_UNINIT buffers, including __uninit kfunc arguments. Allow invalid stack bytes through the output check but leave them invalid when uninitialized stack reads are not allowed. Scrub initialized bytes and scalar spills as possible writes, retaining the existing restrictions on spilled pointers and special stack objects. Retain the output annotation for variable-sized arguments and track their raw-mode eligibility separately. Callers allowed uninitialized stack reads can continue treating the potentially written range as initialized. For constant ranges, defer that initialization until all inputs are checked. Keep prior stack contents live for generic outputs when the caller cannot read uninitialized stack memory. Such calls do not define the entire range, so liveness must preserve initialization facts that remain relevant after the call. Dynptr and iterator constructors still define their storage. Annotate the snprintf, sysctl name, d_path, snprintf_btf and branch-record destinations with MEM_UNINIT, and document that generic __uninit kfuncs may leave bytes untouched. This also changes readback from existing full-writing helpers: without permission to read uninitialized stack memory, programs must initialize those bytes themselves before reading them after a call. Suggested-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-9-memxor@gmail.com --- Documentation/bpf/kfuncs.rst | 21 +++++++++-------- include/linux/bpf.h | 5 +++- include/linux/bpf_verifier.h | 8 ++++--- kernel/bpf/cgroup.c | 2 +- kernel/bpf/helpers.c | 2 +- kernel/bpf/verifier.c | 44 ++++++++++++++++++++++++------------ kernel/trace/bpf_trace.c | 6 ++--- 7 files changed, 54 insertions(+), 34 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index f393c3c3d3b4..c27663cc6cc5 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -164,21 +164,22 @@ suffix should be used. 2.3.3 __uninit Annotation ------------------------- -Use ``__uninit`` on a pointer parameter for an output that the kfunc -initializes without reading its incoming contents. +Use ``__uninit`` on a pointer parameter for an output buffer whose incoming +contents the kfunc does not read. -For generic memory buffers, the kfunc must initialize every byte in the -declared range on every return path, including error returns and struct -padding. The range is determined by the pointed-to type or the associated -``__sz`` or ``__szk`` size argument. +For generic memory buffers, the kfunc may leave bytes untouched, including +on error returns. The writable range is determined by the pointed-to type +or the associated ``__sz`` or ``__szk`` size argument. The annotation does not change the accepted pointer types. A stack-backed struct passed as a generic memory buffer must still be scalar-only. -A stack buffer with a verifier-known constant offset and size may be -uninitialized before the call and is considered initialized afterwards. -For variable offsets or sizes, the usual stack-initialization and -variable-offset restrictions still apply. +Generic stack buffers may be uninitialized before the call. The call does +not make previously uninitialized bytes readable unless the program is +allowed to read uninitialized stack memory (normally requiring +``CAP_PERFMON``). Other callers must initialize those bytes themselves +before reading them. Stack bounds and variable-offset restrictions still +apply. For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a dynptr in the supplied storage. For example:: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5033b934ffd9..fd22db8bc6c5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -781,7 +781,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 9ddbb20ec1e9..92f528c45605 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1547,12 +1547,14 @@ struct ref_obj_desc { }; /* - * Memory arguments a call fills in, indexed by argument slot. The verifier allows the - * stack to be uninitialized if the range is a known constant. Stack slots are marked as - * STACK_MISC by check_mem_access() after all arguments have been checked. + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { u16 mask; + u16 var_size_mask; int size[MAX_BPF_FUNC_ARGS]; }; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 696b27383974..1cb5e6a6ffc1 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2491,7 +2491,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 82402d97ce67..501c7ce35cba 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1130,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c94f1214cc3..7f1cc115456f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6993,10 +6993,11 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; - /* The call will initialize the memory; uninitialized stack allowed */ u32 arg_slot = arg_slot_from_argno(argno); - bool raw_mode = meta && arg_slot < MAX_BPF_FUNC_ARGS && - (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool raw_mode = uninit && env->allow_uninit_stack && + !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); access_size = abs(access_size); @@ -7035,6 +7036,7 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { meta->arg_raw_mem.size[arg_slot] = access_size; return 0; @@ -7054,8 +7056,8 @@ static int check_stack_range_initialized( if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || - (*stype == STACK_INVALID && env->allow_uninit_stack)) { - if (clobber) { + (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { + if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { /* helper can write anything into the stack */ *stype = STACK_MISC; } @@ -7074,8 +7076,11 @@ static int check_stack_range_initialized( } if (*stype == STACK_POISON) { - if (allow_poison) + if (allow_poison) { + if (uninit && env->allow_uninit_stack) + *stype = STACK_MISC; goto mark; + } verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { @@ -7224,12 +7229,12 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, meta->msize_max_value = reg_umax(size_reg); /* - * A variable size does not guarantee that the call initializes the whole - * checked range. Disable raw mode for this output and apply the ordinary - * stack initialization checks, including their privilege exceptions. + * Check variable ranges byte by byte instead of using raw mode. Keep the + * MEM_UNINIT annotation so invalid bytes are accepted without marking them + * initialized when the caller cannot read uninitialized stack memory. */ if (!tnum_is_const(size_reg->var_off)) - meta->arg_raw_mem.mask &= ~BIT(arg_slot_from_argno(mem_argno)); + meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -13730,12 +13735,20 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; const struct bpf_func_proto *fn; enum bpf_arg_type at; + bool full_write; s64 size; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) return S64_MIN; at = fn->arg_type[arg]; + /* + * Generic outputs may leave bytes untouched. Keep prior initialization + * live when the caller cannot read uninitialized bytes. Constructors of + * special objects, such as dynptrs, still define their storage. + */ + full_write = (at & MEM_UNINIT) && + (!arg_type_is_raw_mem(at) || env->allow_uninit_stack); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -13804,7 +13817,7 @@ scan_all_maps: * Size arg is const on each path but differs across merged * paths. MAX_BPF_STACK is a safe upper bound for reads. */ - if (at & MEM_UNINIT) + if (full_write) return 0; return MAX_BPF_STACK; } @@ -13824,10 +13837,10 @@ scan_all_maps: } out: /* - * MEM_UNINIT args are write-only: the helper initializes the - * buffer without reading it. + * Other accesses keep the previous state live, including untouched bytes + * of an unprivileged generic output. */ - if (at & MEM_UNINIT) + if (full_write) return -size; return size; } @@ -13907,7 +13920,8 @@ out: /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) return -size; - if (is_kfunc_arg_uninit(btf, &args[i])) + if (is_kfunc_arg_uninit(btf, &args[i]) && + (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) return -size; return size; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 29260951aa87..195f78db9bda 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -995,7 +995,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1052,7 +1052,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_MEM_SIZE, @@ -1565,7 +1565,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; From a25a61385b04c957277bc421d703e79c8ecf4841 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 21 Sep 2026 04:38:33 +0200 Subject: [PATCH 198/660] selftests/bpf: Cover generic output stack initialization Exercise generic output buffers with and without CAP_PERFMON, using both helpers and __uninit kfuncs. Check that initialized bytes stay readable and lose stale value information, while invalid bytes remain unreadable without permission to read uninitialized stack memory. Cover constant and variable sizes, scalar spills, pointer spills, special stack objects, and privileged variable offsets. Add a kfunc that writes only the first byte of its output. Its runtime tests read preinitialized bytes, checking both the written byte and an untouched tail byte across a liveness checkpoint. Verify that the sysctl name helper and uninitialized fixed and variable-sized kfunc outputs are accepted when their contents are not read back. Update existing __uninit readback expectations and exercise skb_load_bytes with reduced capabilities. Even fully-writing generic outputs now preserve invalid bytes in the verifier, so reading those bytes requires prior initialization by the BPF program. Use map_update_elem inputs for helper_arg_fallback_keeps_scanning. Its original snprintf argument no longer reads the buffer, and a privileged output with an unknown size does not trigger the whole-stack read fallback. A variable-offset key and parent-frame value retain the intended assertion. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-10-memxor@gmail.com --- .../progs/verifier_helper_access_var_len.c | 189 ++++++++++++++++++ .../bpf/progs/verifier_kfunc_uninit.c | 136 +++++++++++++ .../bpf/progs/verifier_kfunc_uninit_multi.c | 6 +- .../selftests/bpf/progs/verifier_live_stack.c | 33 ++- .../selftests/bpf/progs/verifier_raw_stack.c | 4 + .../selftests/bpf/test_kmods/bpf_testmod.c | 7 + .../bpf/test_kmods/bpf_testmod_kfunc.h | 1 + 7 files changed, 356 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index d1452ef6f2f9..e87eb6f221e4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -822,4 +822,193 @@ __naked void bytes_no_leak_init_memory(void) : __clobber_all); } +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_initialized_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_uninitialized_stack(void) +{ + asm volatile ( + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_initialized_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_invalid_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_variable_size(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_variable_invalid(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +__naked void sysctl_partial_spilled_pointer(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "goto +0;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +int sysctl_partial_dynptr(struct bpf_sysctl *ctx) +{ + struct bpf_dynptr ptr; + long long key = 0, *data; + + data = bpf_map_lookup_elem(&map_hash_8b, &key); + if (!data) + return 0; + bpf_dynptr_from_mem(data, sizeof(*data), 0, &ptr); + bpf_sysctl_get_name(ctx, (char *)&ptr, sizeof(ptr), 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__naked void sysctl_partial_variable_offset(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 8;" + "r2 += r10;" + "r2 += -24;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 16);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +int snprintf_partial_output_runtime(struct __sk_buff *ctx) +{ + char buf[16]; + + buf[15] = 42; + bpf_snprintf(buf, sizeof(buf), "ok", NULL, 0); + return buf[15]; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c index f7818303e703..ff2fb36a0260 100644 --- a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c @@ -12,6 +12,7 @@ void __kfunc_btf_root(void) asm volatile ("" : : "r"(&bpf_kfunc_test_uninit_struct), "r"(&bpf_kfunc_test_uninit_mem), + "r"(&bpf_kfunc_test_uninit_partial), "r"(&bpf_kfunc_test_uninit_alias)); } @@ -97,4 +98,139 @@ __naked void uninitialized_input_alias(void) : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); } +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void uninitialized_struct_output_ignored(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void variable_size_uninitialized_output_ignored(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_preserves_initialized_bytes(void) +{ + asm volatile ( + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void uninitialized_sized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void variable_size_uninitialized_output_read(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(1) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") +__naked void fixed_struct_uninitialized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_variable_size(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c index 18ced0d0a3b2..ab3bb81a11fd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c @@ -49,7 +49,7 @@ SEC("tc") __success __retval(42) __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __prepare_priv -__success_unpriv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") __naked void variable_size_preserves_other_output(void) { asm volatile ( @@ -71,7 +71,7 @@ __arch_x86_64 __arch_arm64 __success __retval(42) __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __prepare_priv -__success_unpriv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") __naked void output_after_by_value_argument(void) { asm volatile ( @@ -91,7 +91,7 @@ __arch_x86_64 __arch_arm64 __arch_riscv64 __success __retval(15) __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __prepare_priv -__success_unpriv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") __naked void output_passed_on_stack(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index bc3dfdc1a536..7a1a0670f851 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -21,8 +21,6 @@ struct { __type(value, __u64); } array_map_8b SEC(".maps"); -const char snprintf_u64_fmt[] = "%llu"; - SEC("socket") __log_level(2) __msg("0: (79) r1 = *(u64 *)(r10 -8) ; use: fp0-8") @@ -1947,15 +1945,15 @@ static __used __naked void fwd_parent_key_to_helper(void) /* * Regression for keeping later helper args after a whole-stack fallback - * on an earlier local arg. The first bpf_snprintf() arg is a local + * on an earlier local arg. The bpf_map_update_elem() key is a local * frame-derived pointer with offset-imprecise tracking (`fp1 ?`), which - * conservatively marks the whole local stack live. The fourth arg still + * conservatively marks the whole local stack live. The value arg still * forwards &parent_fp-8 and must contribute nonlocal_use[0]=0:3. */ SEC("socket") __log_level(2) __success -__msg("call bpf_snprintf{{.*}} ; use: fp1-8..-512 fp0-8") +__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-512 fp0-8") __naked void helper_arg_fallback_keeps_scanning(void) { asm volatile ( @@ -1963,32 +1961,33 @@ __naked void helper_arg_fallback_keeps_scanning(void) "*(u64 *)(r10 - 8) = r1;" "r1 = r10;" "r1 += -8;" - "call helper_snprintf_parent_after_local_fallback;" + "call helper_update_parent_after_local_fallback;" "r0 = 0;" "exit;" ::: __clobber_all); } -static __used __naked void helper_snprintf_parent_after_local_fallback(void) +static __used __naked void helper_update_parent_after_local_fallback(void) { asm volatile ( "r6 = r1;" /* save &parent_fp-8 */ "call %[bpf_get_prandom_u32];" "r0 &= 8;" - "r1 = r10;" - "r1 += -16;" - "r1 += r0;" /* local fp, offset-imprecise */ - "r2 = 8;" - "r3 = %[snprintf_u64_fmt] ll;" - "r4 = r6;" /* later arg: parent fp-8 */ - "r5 = 8;" - "call %[bpf_snprintf];" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r2 += r0;" /* local fp, offset-imprecise */ + "r1 = %[array_map_8b] ll;" + "r3 = r6;" /* later arg: parent fp-8 */ + "r4 = 0;" + "call %[bpf_map_update_elem];" "r0 = 0;" "exit;" : : __imm(bpf_get_prandom_u32), - __imm(bpf_snprintf), - __imm_addr(snprintf_u64_fmt) + __imm(bpf_map_update_elem), + __imm_addr(array_map_8b) : __clobber_all); } diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index c689665e07b9..9f0f48ecb421 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c @@ -84,6 +84,8 @@ __naked void skb_load_bytes_zero_len(void) SEC("tc") __description("raw_stack: skb_load_bytes, no init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 8") __naked void skb_load_bytes_no_init(void) { asm volatile (" \ @@ -103,6 +105,8 @@ __naked void skb_load_bytes_no_init(void) SEC("tc") __description("raw_stack: skb_load_bytes, init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv __naked void stack_skb_load_bytes_init(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 211886a8ee87..93847ca6293b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1330,6 +1330,12 @@ __bpf_kfunc void bpf_kfunc_test_uninit_mem(void *out__uninit, u32 out__sz) memset(out__uninit, 0x2a, out__sz); } +__bpf_kfunc void bpf_kfunc_test_uninit_partial(void *out__uninit, u32 out__sz) +{ + if (out__sz) + *(u8 *)out__uninit = 42; +} + __bpf_kfunc int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) { int value = get_unaligned(in); @@ -1788,6 +1794,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_struct) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_mem) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_partial) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_alias) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_multi) BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_pair) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 524f2cb9bdf4..67c02a421d13 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -291,6 +291,7 @@ void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) __ksym; void bpf_kfunc_test_uninit_mem(void *out__uninit, __u32 out__sz) __ksym; +void bpf_kfunc_test_uninit_partial(void *out__uninit, __u32 out__sz) __ksym; int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) __ksym; void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, __u32 b__sz) __ksym; void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) __ksym; From 7a6cef39b76043e33c04f2510aa1a59c540bb1d6 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 21 Sep 2026 04:38:34 +0200 Subject: [PATCH 199/660] bpf: Check read access for helper input/output buffers MEM_WRITE without MEM_UNINIT denotes memory that is read as well as written. Helper argument checking requests only BPF_WRITE for it, which checks stack initialization but omits read permission checks on map values. For example, bpf_check_mtu() reads its mtu_len argument before overwriting it, but the verifier permits that argument to point into a BPF_F_WRONLY_PROG map. bpf_fib_lookup() and bpf_load_hdr_opt() are affected the same way. Derive generic memory access from argument flags: read-only for inputs, write-only for MEM_WRITE | MEM_UNINIT, and read/write for MEM_WRITE alone. Partial-output helpers such as bpf_snprintf() now carry MEM_UNINIT, so their write-only map destinations remain valid. Generated kfunc prototypes already encode the same distinction, which retires the kfunc-specific access override. Signed-off-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-11-memxor@gmail.com --- kernel/bpf/verifier.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7f1cc115456f..d62c0f74cff5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8907,6 +8907,19 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } +/* + * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output + * whose incoming contents are not read. + */ +static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type) +{ + if (!(arg_type & MEM_WRITE)) + return BPF_READ; + if (arg_type & MEM_UNINIT) + return BPF_WRITE; + return BPF_READ | BPF_WRITE; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) @@ -9199,10 +9212,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (!(arg_type & MEM_FIXED_SIZE)) break; - access_type = arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ; - /* Ordinary kfunc buffers are input/output; __uninit buffers are outputs. */ - if (meta->btf && !(arg_type & MEM_UNINIT)) - access_type = BPF_READ | BPF_WRITE; + access_type = func_arg_access_type(arg_type); err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); if (err < 0) { @@ -9248,9 +9258,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (meta->btf && bpf_register_is_null(buff_reg)) break; - access_type = fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ; - if (meta->btf && !(fn->arg_type[arg - 1] & MEM_UNINIT)) - access_type = BPF_READ | BPF_WRITE; + access_type = func_arg_access_type(fn->arg_type[arg - 1]); zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; From ad36589c3f39f982010987448b90d580e0f455d3 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 21 Sep 2026 04:38:35 +0200 Subject: [PATCH 200/660] selftests/bpf: Cover helper memory access permissions Check that fixed-size and sized helper input/output buffers require both read and write permission. Cover the MTU and FIB helpers, including read-only and write-only rejection and read/write positive controls. Exercise the XDP prototypes and the sock_ops header-option input/output argument as well. Keep write-only maps usable as destinations for partial-output helpers bpf_snprintf() and bpf_sysctl_get_name(), while rejecting a read-only snprintf destination. Also retain a write-only-map destination for bpf_get_current_comm(), whose output is annotated MEM_UNINIT and fully initialized by the helper. Signed-off-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260921023843.411943-12-memxor@gmail.com --- .../progs/verifier_helper_access_var_len.c | 143 ++++++++++++++++++ .../selftests/bpf/progs/verifier_mtu.c | 88 +++++++++++ 2 files changed, 231 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index e87eb6f221e4..2782faf0a528 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ /* Converted from tools/testing/selftests/bpf/verifier/helper_access_var_len.c */ #include @@ -822,6 +823,148 @@ __naked void bytes_no_leak_init_memory(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_ro SEC(".maps"); + +SEC("tc") +__failure __msg("read from map forbidden") +int writeonly_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__success +int readwrite_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_rw, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int readonly_sized_output(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_ro, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_sized_input(struct xdp_md *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return XDP_PASS; +} + +SEC("sockops") +__failure __msg("read from map forbidden") +int writeonly_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("sockops") +__success +int readwrite_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_rw, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("tc") +__success +int snprintf_writeonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int snprintf_readonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_ro, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +int sysctl_writeonly_output(struct bpf_sysctl *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_sysctl_get_name(ctx, buf, 16, 0); + return 0; +} + SEC("cgroup/sysctl") __success __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) diff --git a/tools/testing/selftests/bpf/progs/verifier_mtu.c b/tools/testing/selftests/bpf/progs/verifier_mtu.c index 256956ea1ac5..2f71f70302e4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_mtu.c +++ b/tools/testing/selftests/bpf/progs/verifier_mtu.c @@ -17,4 +17,92 @@ int tc_uninit_mtu(struct __sk_buff *ctx) return TCX_PASS; } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_mtu_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_ro SEC(".maps"); + +SEC("tc/ingress") +__failure __msg("read from map forbidden") +int tc_writeonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_readwrite_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_rw, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_mtu(struct xdp_md *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return XDP_PASS; +} + +SEC("tc/ingress") +__failure __msg("write into map forbidden") +int tc_readonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_ro, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_writeonly_output(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *out; + + out = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (out) + bpf_get_current_comm(out, sizeof(*out)); + return TCX_PASS; +} + char LICENSE[] SEC("license") = "GPL"; From a885470df10b1c0dbc011177c19d3fd3bb8af2e9 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 16 Sep 2026 00:14:50 +0800 Subject: [PATCH 201/660] bpftool: Compute map size of light skeletons at runtime bpftool rounds memory-mapped data map sizes to the host page size when generating a light skeleton. The generated code therefore uses a 64K mapping size when bpftool runs on a 64K-page host, even if the skeleton runs on a 4K-page target. The target rejects the oversized map mmap(), causing failure of loading the light skeleton. When try to run 64K-page selftests on 4K-page VM, the error message does not provide the reason about page size. test_atomics:PASS:atomics skeleton open 0 nsec test_atomics:FAIL:atomics skeleton load unexpected error: -12 (errno 22) #15 atomics:FAIL Pass the original map value size and max entries to the generated code and round mmap size to the runtime page size in the user-space light skeleton helpers. This keeps generated light skeletons independent of the build host page size. Fixes: d510296d331a ("bpftool: Use syscall/loader program in "prog load" and "gen skeleton" command.") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Acked-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20260915161450.96249-1-leon.hwang@linux.dev --- tools/bpf/bpftool/gen.c | 26 ++++++------------- tools/lib/bpf/skel_internal.h | 48 +++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index a50540ef6521..ee680f7c2116 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -500,16 +500,6 @@ static void print_hex(const char *data, int data_sz) } } -static size_t bpf_map_mmap_sz(const struct bpf_map *map) -{ - long page_sz = sysconf(_SC_PAGE_SIZE); - size_t map_sz; - - map_sz = (size_t)roundup(bpf_map__value_size(map), 8) * bpf_map__max_entries(map); - map_sz = roundup(map_sz, page_sz); - return map_sz; -} - /* Emit type size asserts for all top-level fields in memory-mapped internal maps. */ static void codegen_asserts(struct bpf_object *obj, const char *obj_name) { @@ -686,8 +676,8 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) if (!get_map_ident(map, ident, sizeof(ident))) continue; if (is_skel_data(map, ident, sizeof(ident))) - printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n", - ident, bpf_map_mmap_sz(map)); + printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$u, %3$u);\n", + ident, bpf_map__value_size(map), bpf_map__max_entries(map)); codegen("\ \n\ skel_closenz(skel->maps.%1$s.map_fd); \n\ @@ -771,13 +761,13 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h \n\ \"; \n\ \n\ - skel->%1$s = (__typeof__(skel->%1$s))skel_prep_map_data((void *)data, %2$zd,\n\ + skel->%1$s = (__typeof__(skel->%1$s))skel_prep_map_data((void *)data, %2$u, %3$u,\n\ sizeof(data) - 1);\n\ if (!skel->%1$s) \n\ goto cleanup; \n\ skel->maps.%1$s.initial_value = (__u64) (long) skel->%1$s;\n\ } \n\ - ", ident, bpf_map_mmap_sz(map)); + ", ident, bpf_map__value_size(map), bpf_map__max_entries(map)); } codegen("\ \n\ @@ -871,14 +861,14 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { codegen("\ \n\ - err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$zd);\n\ + err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$u, %3$u);\n\ if (err) \n\ return err; \n\ #ifdef __KERNEL__ \n\ skel->%1$s = NULL; \n\ #endif \n\ ", - ident, bpf_map_mmap_sz(map)); + ident, bpf_map__value_size(map), bpf_map__max_entries(map)); continue; } @@ -890,11 +880,11 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h codegen("\ \n\ skel->%1$s = (__typeof__(skel->%1$s))skel_finalize_map_data(&skel->maps.%1$s.initial_value,\n\ - %2$zd, %3$s, skel->maps.%1$s.map_fd);\n\ + %2$u, %3$u, %4$s, skel->maps.%1$s.map_fd);\n\ if (!skel->%1$s) \n\ return -ENOMEM; \n\ ", - ident, bpf_map_mmap_sz(map), mmap_flags); + ident, bpf_map__value_size(map), bpf_map__max_entries(map), mmap_flags); } codegen("\ \n\ diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 1f3f332dffbe..9e16b8e8cb7c 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -11,6 +11,7 @@ #include #else #include +#include #include #include #include @@ -127,7 +128,7 @@ static inline void skel_free(const void *p) * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value * depending on bpf_loader_ctx->flags. */ -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) +static inline void skel_free_map_data(void *p, __u64 addr, size_t val_sz, __u32 max_entries) { if (addr != ~0ULL) kvfree(p); @@ -138,18 +139,20 @@ static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) */ } -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) +static inline void *skel_prep_map_data(const void *val, size_t val_sz, __u32 max_entries, + size_t data_sz) { void *addr; - addr = kvmalloc(val_sz, GFP_KERNEL); + addr = kvmalloc(data_sz, GFP_KERNEL); if (!addr) return NULL; - memcpy(addr, val, val_sz); + memcpy(addr, val, data_sz); return addr; } -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) +static inline void *skel_finalize_map_data(__u64 *init_val, size_t val_sz, __u32 max_entries, + int flags, int fd) { struct bpf_map *map; void *addr = NULL; @@ -172,9 +175,10 @@ out: return addr; } -static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t val_sz, __u32 max_entries) { - (void)sz; + (void)val_sz; + (void)max_entries; kvfree(p); *init_val = ~0ULL; @@ -193,25 +197,39 @@ static inline void skel_free(void *p) free(p); } -static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) +static inline size_t skel_map_mmap_sz(size_t val_sz, __u32 max_entries) { - munmap(p, sz); + const long page_sz = sysconf(_SC_PAGE_SIZE); + size_t mmap_sz; + + mmap_sz = roundup(val_sz, 8) * max_entries; + mmap_sz = roundup(mmap_sz, page_sz); + return mmap_sz; } -static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) +static inline void skel_free_map_data(void *p, __u64 addr, size_t val_sz, __u32 max_entries) { + munmap(p, skel_map_mmap_sz(val_sz, max_entries)); +} + +static inline void *skel_prep_map_data(const void *val, size_t val_sz, __u32 max_entries, + size_t data_sz) +{ + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); void *addr; addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (addr == (void *) -1) return NULL; - memcpy(addr, val, val_sz); + memcpy(addr, val, data_sz); return addr; } -static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) +static inline void *skel_finalize_map_data(__u64 *init_val, size_t val_sz, __u32 max_entries, + int flags, int fd) { + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); void *addr; addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); @@ -220,11 +238,13 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return addr; } -static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t val_sz, __u32 max_entries) { + size_t mmap_sz = skel_map_mmap_sz(val_sz, max_entries); + (void)init_val; - if (mprotect(p, sz, PROT_READ)) + if (mprotect(p, mmap_sz, PROT_READ)) return -errno; return 0; } From 7f2b0b57a8d7dc0772ab998007d5d8e8f1f94966 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 15 Sep 2026 23:30:09 +0200 Subject: [PATCH 202/660] bpf: Add printf attributes Building with extra warnings enabled shows a missing printf attribute: kernel/bpf/diagnostics.c: In function 'bpf_diag_source': kernel/bpf/diagnostics.c:828:9: error: function 'bpf_diag_source' might be a candidate for 'gnu_printf' format attribute [-Werror=suggest-attribute=format] 828 | msg = bpf_diag_vfmt(env, fmt, args); Add these to shut up the warning and allow the compiler to check the format strings. Fixes: b9c5d822f677 ("bpf: Add source and instruction diagnostic context") Signed-off-by: Arnd Bergmann Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260915213123.2361066-1-arnd@kernel.org --- kernel/bpf/diagnostics.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 787932f92cdf..5ecfa86ed49f 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -794,8 +794,9 @@ static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, } } -static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, - const char *fmt, ...) +static __printf(4, 5) void bpf_diag_source(struct bpf_verifier_env *env, + u32 insn_idx, const char *label, + const char *fmt, ...) { struct bpf_diag_scratch *scratch; struct bpf_linfo_source *source_lines; From 6892723c5eb612c37ad57f50efec54406569a9f1 Mon Sep 17 00:00:00 2001 From: Luis Vieira Date: Wed, 16 Sep 2026 18:23:35 -0300 Subject: [PATCH 203/660] libbpf: Validate ELF symbol table entry size elf_sym_iter_new() calculates the number of symbols by dividing the symbol table data size by sh_entsize. A malformed ELF file with a zero sh_entsize causes a division by zero. Reject symbol table sections with a zero entry size before calculating the number of symbols. Signed-off-by: Luis Vieira Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260916-libbpf-elf-entsize-fix-v1-1-cf181da33eab@gmail.com --- tools/lib/bpf/elf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index fe136d025967..700c3911a0d5 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -115,7 +115,7 @@ static int elf_sym_iter_new(struct elf_sym_iter *iter, return -ENOENT; } - if (!gelf_getshdr(scn, &sh)) + if (!gelf_getshdr(scn, &sh) || !sh.sh_entsize) return -EINVAL; iter->strtabidx = sh.sh_link; From 80ed1435d97d1d77d706078f2f25c3d8673aac0a Mon Sep 17 00:00:00 2001 From: Zhixing Chen Date: Thu, 17 Sep 2026 17:39:28 +0800 Subject: [PATCH 204/660] selftests/bpf: Guard link cleanup in fexit_bpf2bpf test_fexit_bpf2bpf_common() can jump to the common cleanup path before the link array is allocated, for example if bpf_prog_get_info_by_fd() fails. The cleanup loop still indexes link[i] unconditionally, which can dereference a NULL pointer and hide the original failure. Guard the loop so the test reports the original failure instead. Signed-off-by: Zhixing Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260917093928.48495-1-running910@gmail.com --- tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 6262c81cdaa9..d2f2dcc0f2e1 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -164,8 +164,10 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, goto close_prog; close_prog: - for (i = 0; i < prog_cnt; i++) - bpf_link__destroy(link[i]); + if (link) { + for (i = 0; i < prog_cnt; i++) + bpf_link__destroy(link[i]); + } bpf_object__close(obj); bpf_object__close(tgt_obj); free(link); From 8715b13c1a800f34f9d8177837e5244ac8288f60 Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Fri, 18 Sep 2026 10:43:00 +0800 Subject: [PATCH 205/660] selftests/bpf: Clean up child when task_work__open() fails task_work_run() forks a child that blocks reading a pipe until the parent wakes it. If task_work__open() fails afterwards, the parent returns without closing either pipe fd or waiting for the child. Because the parent retains the write end, the child remains blocked in read() until test_progs exits. Route this failure through the common cleanup path. At this point cleanup is safe: pe_fd is -1, link is NULL, task_work__destroy() accepts NULL, and the pid > 0 branch closes the read end, wakes the child, closes the write end and waits for it. This is the last early return after a successful fork. The fork failure path already closes both pipe fds after commit 5730dacb3f17 ("selftests/bpf: Task_work selftest cleanup fixes"). Fixes: 39fd74dfd5d2 ("selftests/bpf: BPF task work scheduling tests") Signed-off-by: Yun Lu Signed-off-by: Andrii Nakryiko Acked-by: Mykyta Yatsenko Link: https://lore.kernel.org/bpf/20260918024300.18321-1-luyun_611@163.com --- tools/testing/selftests/bpf/prog_tests/test_task_work.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_work.c b/tools/testing/selftests/bpf/prog_tests/test_task_work.c index 774b31a5f6ca..fe0cb1702eaa 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_work.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_work.c @@ -85,7 +85,7 @@ static void task_work_run(const char *prog_name, const char *map_name) skel = task_work__open(); if (!ASSERT_OK_PTR(skel, "task_work__open")) - return; + goto cleanup; bpf_object__for_each_program(prog, skel->obj) { bpf_program__set_autoload(prog, false); From 32a7b17bc8e7a198dc5cfb4601e1eef819e8e316 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Mon, 14 Sep 2026 22:29:04 -0400 Subject: [PATCH 206/660] bpf, parisc: Add support for BPF_SDIV and BPF_SMOD in the parisc64 JIT emit_call_libgcc_ll() zero-extends the ALU32 operands and calls the unsigned hppa_div64()/hppa_div64_rem() helpers regardless of the BPF instruction's signedness, so the parisc64 JIT does not implement signed BPF_SDIV and BPF_SMOD (off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands. Add hppa_sdiv64()/hppa_sdiv64_rem() wrapping div64_s64(), thread is_signed = (insn->off == 1) through the div/mod emit sites, and on the signed path sign-extend the ALU32 operands (and the immediate divisor) instead of zero-extending them before calling the signed helpers. bpf_do_misc_fixups() rewrites the zero-divisor and INT_MIN/-1 cases out of the instruction stream before the JIT runs. Signed-off-by: Nicholas Dudar Signed-off-by: Daniel Borkmann Tested-by: Helge Deller Acked-by: Helge Deller Link: https://lore.kernel.org/bpf/20260915022905.1535566-2-main.kalliope@gmail.com --- arch/parisc/net/bpf_jit.h | 2 ++ arch/parisc/net/bpf_jit_comp64.c | 35 ++++++++++++++++++++++++-------- arch/parisc/net/bpf_jit_core.c | 14 +++++++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/arch/parisc/net/bpf_jit.h b/arch/parisc/net/bpf_jit.h index 8b8896959f04..074115c93c90 100644 --- a/arch/parisc/net/bpf_jit.h +++ b/arch/parisc/net/bpf_jit.h @@ -467,6 +467,8 @@ static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t) u64 hppa_div64(u64 div, u64 divisor); u64 hppa_div64_rem(u64 div, u64 divisor); +u64 hppa_sdiv64(u64 div, u64 divisor); +u64 hppa_sdiv64_rem(u64 div, u64 divisor); /* Helper functions that emit HPPA instructions when possible. */ diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c index 54b0d5e25e02..c326fa737ec8 100644 --- a/arch/parisc/net/bpf_jit_comp64.c +++ b/arch/parisc/net/bpf_jit_comp64.c @@ -502,14 +502,19 @@ static void emit_call(u64 addr, bool fixed, struct hppa_jit_context *ctx) emit_hppa_copy(HPPA_REG_RET0, regmap[BPF_REG_0], ctx); } -static void emit_call_libgcc_ll(void *func, const s8 arg0, - const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) +static void emit_call_libgcc_ll(void *func, const s8 arg0, const s8 arg1, + u8 opcode, bool is_signed, struct hppa_jit_context *ctx) { u64 func_addr; if (BPF_CLASS(opcode) == BPF_ALU) { - emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); - emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + if (is_signed) { + emit_hppa64_sext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_sext32(arg1, HPPA_REG_ARG1, ctx); + } else { + emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + } } else { emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx); emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx); @@ -600,6 +605,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, u8 rd = -1, rs = -1, code = insn->code; s16 off = insn->off; s32 imm = insn->imm; + bool is_signed; + void *func; init_regs(&rd, &rs, insn, ctx); @@ -656,29 +663,39 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, fallthrough; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: - emit_call_libgcc_ll(__muldi3, rd, rs, code, ctx); + emit_call_libgcc_ll(__muldi3, rd, rs, code, false, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: - emit_call_libgcc_ll(&hppa_div64, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 172770132440..a2f93202a2ef 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -190,3 +190,17 @@ u64 hppa_div64_rem(u64 div, u64 divisor) div64_u64_rem(div, divisor, &rem); return rem; } + +u64 hppa_sdiv64(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)sdiv; +} + +u64 hppa_sdiv64_rem(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)((s64)div - sdiv * (s64)divisor); +} From dbda00ac6f0a17d4c9d446c5c66d53ae9f3a75fc Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Mon, 14 Sep 2026 22:29:05 -0400 Subject: [PATCH 207/660] bpf, parisc: Add support for BPF_SDIV and BPF_SMOD in the parisc32 JIT emit_alu_r32() and emit_alu_r64() call the unsigned millicode ($$divU/$$remU) and libgcc-style (hppa_div64()/hppa_div64_rem()) divide/remainder helpers for BPF_DIV and BPF_MOD regardless of the BPF instruction's signedness, so the parisc32 JIT does not implement signed BPF_SDIV and BPF_SMOD (off == 1). Signed ALU32 and ALU64 div/mod get an unsigned quotient and remainder rather than the verifier's and the interpreter's signed result for negative operands. Thread is_signed = (insn->off == 1) through emit_alu_r32()/ emit_alu_r64(), and on the signed path use the $$divI/$$remI signed millicode routines (32-bit) or the hppa_sdiv64()/hppa_sdiv64_rem() helpers added in patch 1 (64-bit) instead of their unsigned counterparts. $$divI and $$remI are extern-declared and exported next to $$divU/$$remU in arch/parisc/kernel/parisc_ksyms.c. bpf_do_misc_fixups() rewrites the zero-divisor and INT_MIN/-1 cases out of the instruction stream before the JIT runs. Signed-off-by: Nicholas Dudar Signed-off-by: Daniel Borkmann Tested-by: Helge Deller Acked-by: Helge Deller Link: https://lore.kernel.org/bpf/20260915022905.1535566-3-main.kalliope@gmail.com --- arch/parisc/net/bpf_jit_comp32.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c index 5ff0cf925fe9..eea0a34a3bfe 100644 --- a/arch/parisc/net/bpf_jit_comp32.c +++ b/arch/parisc/net/bpf_jit_comp32.c @@ -321,6 +321,8 @@ static void bpf_put_reg32(const s8 *reg, const s8 *src, extern void $$mulI(void); extern void $$divU(void); extern void $$remU(void); +extern void $$divI(void); +extern void $$remI(void); static void emit_call_millicode(void *func, const s8 arg0, const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) @@ -562,12 +564,14 @@ static void emit_alu_i32(const s8 *dst, s32 imm, } static void emit_alu_r64(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; const s8 *rd; const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + void *func; if (op == BPF_MOV) rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); @@ -604,10 +608,12 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, emit_call_libgcc_ll(__muldi3, rd, rs, op, ctx); break; case BPF_DIV: - emit_call_libgcc_ll(&hppa_div64, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_MOD: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_LSH: emit_call_libgcc_ll(__ashldi3, rd, rs, op, ctx); @@ -630,7 +636,8 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; @@ -666,10 +673,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit_call_millicode($$mulI, lo(rd), lo(rs), op, ctx); break; case BPF_DIV: - emit_call_millicode($$divU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$divI : $$divU, + lo(rd), lo(rs), op, ctx); break; case BPF_MOD: - emit_call_millicode($$remU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$remI : $$remU, + lo(rd), lo(rs), op, ctx); break; case BPF_LSH: emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); @@ -1163,12 +1172,12 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r64(dst, src, ctx, BPF_OP(code)); + emit_alu_r64(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = -dst */ case BPF_ALU64 | BPF_NEG: - emit_alu_r64(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r64(dst, tmp2, ctx, BPF_OP(code), false); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -1211,7 +1220,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = dst OP imm */ @@ -1236,7 +1245,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, BPF_OP(code), false); break; /* dst = BSWAP##imm(dst) */ From b5dc1b184f32077f45c9d31514f166b6388fa57f Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Mon, 21 Sep 2026 16:35:55 +0900 Subject: [PATCH 208/660] bpftool: Read ring buffer maps with event_pipe map event_pipe only accepts perf event arrays, leaving no built-in way to inspect records produced through the BPF ring buffer API. Extend it to consume queued and live BPF_MAP_TYPE_RINGBUF records in plain or JSON output, retaining the existing perf event array path. Ring buffers have a shared consumer position and no implicit CPU or timestamp. Document that this command consumes records rather than observing them passively, and reject perf-only CPU/index selectors. A producer can keep ring_buffer__poll() busy after a stop signal, so return -EINTR from the record callback when stopping. Keep stdio out of the shared signal handler and preserve callback output errors. Signed-off-by: Tianyi Chen Signed-off-by: Andrii Nakryiko Tested-by: Quentin Monnet Link: https://github.com/libbpf/bpftool/issues/54 Link: https://lore.kernel.org/bpf/20260921073556.99421-2-hi@tychen.cc --- .../bpf/bpftool/Documentation/bpftool-map.rst | 14 ++- tools/bpf/bpftool/bash-completion/bpftool | 59 +++++++------ tools/bpf/bpftool/map_perf_ring.c | 87 ++++++++++++++----- 3 files changed, 112 insertions(+), 48 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst index 5daf3de5c744..e4ed7e701c9e 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-map.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst @@ -120,7 +120,8 @@ bpftool map pin *MAP* *FILE* character ('.'), which is reserved for future extensions of *bpffs*. bpftool map event_pipe *MAP* [cpu *N* index *M*] - Read events from a **BPF_MAP_TYPE_PERF_EVENT_ARRAY** map. + Read events from a **BPF_MAP_TYPE_PERF_EVENT_ARRAY** or + **BPF_MAP_TYPE_RINGBUF** map. Install perf rings into a perf event array map and dump output of any **bpf_perf_event_output**\ () call in the kernel. By default read the @@ -134,6 +135,17 @@ bpftool map event_pipe *MAP* [cpu *N* index *M*] existing ring. Any other application will stop receiving events if it installed its rings earlier. + For a ring buffer map, consume records submitted by BPF programs, including + records already queued before the command starts. **cpu** and **index** + are not supported. Each record is printed in full, including embedded zero + bytes. Plain output reports the record size followed by hexadecimal + bytes; JSON output contains **size** and **data** fields, with **data** an + array of byte values. Ring buffer records have no implicit CPU or timestamp. + + Consuming a ring buffer advances its shared consumer position, so this + command must not run alongside another consumer of the same map. + **BPF_MAP_TYPE_USER_RINGBUF** maps are not supported. + bpftool map peek *MAP* Peek next value in the queue or stack. diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 75cbcb512eba..c9e8761e4ef2 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -50,12 +50,19 @@ _bpftool_get_map_ids() command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) ) } -# Takes map type and adds matching map ids to the list of suggestions. -_bpftool_get_map_ids_for_type() +# Takes map types and prints a pattern matching their JSON type fields. +_bpftool_build_type_pattern() { - local type="$1" + local IFS='|' + printf '"type": "(%s)"\n' "$*" +} + +# Takes map types and adds matching map ids to the list of suggestions. +_bpftool_get_map_ids_for_types() +{ + local types=$(_bpftool_build_type_pattern "$@") COMPREPLY+=( $( compgen -W "$( bpftool -jp map 2>&1 | \ - command grep -C2 "$type" | \ + command grep -C2 -E "$types" | \ command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) ) } @@ -65,12 +72,12 @@ _bpftool_get_map_names() command sed -n 's/.*"name": \(.*\),$/\1/p' )" -- "$cur" ) ) } -# Takes map type and adds matching map names to the list of suggestions. -_bpftool_get_map_names_for_type() +# Takes map types and adds matching map names to the list of suggestions. +_bpftool_get_map_names_for_types() { - local type="$1" + local types=$(_bpftool_build_type_pattern "$@") COMPREPLY+=( $( compgen -W "$( bpftool -jp map 2>&1 | \ - command grep -C2 "$type" | \ + command grep -C2 -E "$types" | \ command sed -n 's/.*"name": \(.*\),$/\1/p' )" -- "$cur" ) ) } @@ -139,7 +146,7 @@ _bpftool_map_guess_map_type() local keyword idx ref="" for (( idx=3; idx < ${#words[@]}-1; idx++ )); do case "${words[$((idx-2))]}" in - lookup|update) + lookup|update|event_pipe) keyword=${words[$((idx-1))]} ref=${words[$((idx))]} ;; @@ -177,10 +184,10 @@ _bpftool_map_update_get_id() if [[ $value -eq 0 ]]; then case "$command" in push) - _bpftool_get_map_ids_for_type stack + _bpftool_get_map_ids_for_types stack ;; enqueue) - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types queue ;; *) _bpftool_get_map_ids @@ -223,10 +230,10 @@ _bpftool_map_update_get_name() if [[ $value -eq 0 ]]; then case "$command" in push) - _bpftool_get_map_names_for_type stack + _bpftool_get_map_names_for_types stack ;; enqueue) - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types queue ;; *) _bpftool_get_map_names @@ -617,10 +624,10 @@ _bpftool() COMPREPLY=( $( compgen -W "$STRUCT_OPS_TYPE" -- "$cur" ) ) ;; id) - _bpftool_get_map_ids_for_type struct_ops + _bpftool_get_map_ids_for_types struct_ops ;; name) - _bpftool_get_map_names_for_type struct_ops + _bpftool_get_map_names_for_types struct_ops ;; esac return 0 @@ -679,14 +686,13 @@ _bpftool() id) case "$command" in peek) - _bpftool_get_map_ids_for_type stack - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types stack queue ;; pop) - _bpftool_get_map_ids_for_type stack + _bpftool_get_map_ids_for_types stack ;; dequeue) - _bpftool_get_map_ids_for_type queue + _bpftool_get_map_ids_for_types queue ;; *) _bpftool_get_map_ids @@ -697,14 +703,13 @@ _bpftool() name) case "$command" in peek) - _bpftool_get_map_names_for_type stack - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types stack queue ;; pop) - _bpftool_get_map_names_for_type stack + _bpftool_get_map_names_for_types stack ;; dequeue) - _bpftool_get_map_names_for_type queue + _bpftool_get_map_names_for_types queue ;; *) _bpftool_get_map_names @@ -878,11 +883,11 @@ _bpftool() return 0 ;; id) - _bpftool_get_map_ids_for_type perf_event_array + _bpftool_get_map_ids_for_types perf_event_array ringbuf return 0 ;; name) - _bpftool_get_map_names_for_type perf_event_array + _bpftool_get_map_names_for_types perf_event_array ringbuf return 0 ;; cpu) @@ -892,7 +897,9 @@ _bpftool() return 0 ;; *) - _bpftool_once_attr 'cpu index' + if [[ $(_bpftool_map_guess_map_type) != ringbuf ]]; then + _bpftool_once_attr 'cpu index' + fi return 0 ;; esac diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c index bcb767e2d673..5878938cf8d9 100644 --- a/tools/bpf/bpftool/map_perf_ring.c +++ b/tools/bpf/bpftool/map_perf_ring.c @@ -27,7 +27,7 @@ #define MMAP_PAGE_CNT 16 -static volatile bool stop; +static volatile sig_atomic_t stop; struct perf_event_sample { struct perf_event_header header; @@ -44,7 +44,6 @@ struct perf_event_lost { static void int_exit(int signo) { - fprintf(stderr, "Stopping...\n"); stop = true; } @@ -107,6 +106,27 @@ print_bpf_output(void *private_data, int cpu, struct perf_event_header *event) return LIBBPF_PERF_EVENT_CONT; } +static int print_ringbuf_output(void *ctx, void *data, size_t size) +{ + if (json_output) { + jsonw_start_object(json_wtr); + jsonw_uint_field(json_wtr, "size", size); + jsonw_name(json_wtr, "data"); + print_data_json(data, size); + jsonw_end_object(json_wtr); + } else { + printf("== size: %zu =====\n", size); + fprint_hex(stdout, data, size, " "); + printf("\n"); + } + + if (fflush(stdout)) + return errno ? -errno : -EIO; + + /* A producer can keep poll() busy even after a signal arrives. */ + return stop ? -EINTR : 0; +} + int do_event_pipe(int argc, char **argv) { struct perf_event_attr perf_attr = { @@ -123,18 +143,27 @@ int do_event_pipe(int argc, char **argv) .cpu = -1, .idx = -1, }; - struct perf_buffer *pb; + struct perf_buffer *pb = NULL; + struct ring_buffer *rb = NULL; __u32 map_info_len; int err, map_fd; + stop = false; map_info_len = sizeof(map_info); map_fd = map_parse_fd_and_info(&argc, &argv, &map_info, &map_info_len, 0); if (map_fd < 0) return -1; - if (map_info.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { - p_err("map is not a perf event array"); + if (map_info.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && + map_info.type != BPF_MAP_TYPE_RINGBUF) { + p_err("map is not a perf event array or ring buffer"); + goto err_close_map; + } + + if (map_info.type == BPF_MAP_TYPE_RINGBUF && argc && + (is_prefix(*argv, "cpu") || is_prefix(*argv, "index"))) { + p_err("ring buffer maps do not support cpu or index arguments"); goto err_close_map; } @@ -184,15 +213,24 @@ int do_event_pipe(int argc, char **argv) ctx.idx = 0; } - opts.cpu_cnt = ctx.all_cpus ? 0 : 1; - opts.cpus = &ctx.cpu; - opts.map_keys = &ctx.idx; - pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, - print_bpf_output, &ctx, &opts); - if (!pb) { - p_err("failed to create perf buffer: %s (%d)", - strerror(errno), errno); - goto err_close_map; + if (map_info.type == BPF_MAP_TYPE_RINGBUF) { + rb = ring_buffer__new(map_fd, print_ringbuf_output, NULL, NULL); + if (!rb) { + p_err("failed to create ring buffer: %s (%d)", + strerror(errno), errno); + goto err_close_map; + } + } else { + opts.cpu_cnt = ctx.all_cpus ? 0 : 1; + opts.cpus = &ctx.cpu; + opts.map_keys = &ctx.idx; + pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, + print_bpf_output, &ctx, &opts); + if (!pb) { + p_err("failed to create perf buffer: %s (%d)", + strerror(errno), errno); + goto err_close_map; + } } signal(SIGINT, int_exit); @@ -202,25 +240,32 @@ int do_event_pipe(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); + err = 0; while (!stop) { - err = perf_buffer__poll(pb, 200); + err = rb ? ring_buffer__poll(rb, 200) : perf_buffer__poll(pb, 200); if (err < 0 && err != -EINTR) { - p_err("perf buffer polling failed: %s (%d)", - strerror(errno), errno); - goto err_close_pb; + p_err("%s buffer polling failed: %s (%d)", + rb ? "ring" : "perf", strerror(-err), -err); + break; } + err = 0; } + if (stop) + fprintf(stderr, "Stopping...\n"); if (json_output) jsonw_end_array(json_wtr); + if (fflush(stdout)) { + p_err("failed to write events: %s", strerror(errno)); + err = -1; + } + ring_buffer__free(rb); perf_buffer__free(pb); close(map_fd); - return 0; + return err < 0 ? -1 : 0; -err_close_pb: - perf_buffer__free(pb); err_close_map: close(map_fd); return -1; From b09784655ecc3efdf817a6e60c0b609e9f0a52c1 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Mon, 21 Sep 2026 16:35:56 +0900 Subject: [PATCH 209/660] selftests/bpf: Cover bpftool ring buffer event_pipe Produce known ring buffer records and check complete plain, JSON and pretty JSON output. Exercise ID and pinned map selection, SIGINT and SIGTERM shutdown, empty streams and invalid map types or selectors. Also produce a perf event sample and check its existing header and raw payload output. Use a payload whose size plus the raw sample length field is aligned to eight bytes so the expected bytes exclude implicit perf padding. Signed-off-by: Tianyi Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260921073556.99421-3-hi@tychen.cc --- tools/testing/selftests/bpf/bpftool_helpers.c | 2 +- tools/testing/selftests/bpf/bpftool_helpers.h | 1 + .../bpf/prog_tests/bpftool_ringbuf.c | 417 ++++++++++++++++++ .../selftests/bpf/progs/bpftool_ringbuf.c | 47 ++ 4 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c create mode 100644 tools/testing/selftests/bpf/progs/bpftool_ringbuf.c diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index c49fdd90eb03..9380b24bbf52 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c @@ -11,7 +11,7 @@ #define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool" -static int detect_bpftool_path(char *buffer, size_t size) +int detect_bpftool_path(char *buffer, size_t size) { char tmp[PATH_MAX]; const char *env_path; diff --git a/tools/testing/selftests/bpf/bpftool_helpers.h b/tools/testing/selftests/bpf/bpftool_helpers.h index dec1ba201410..17e3be8e7073 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.h +++ b/tools/testing/selftests/bpf/bpftool_helpers.h @@ -7,5 +7,6 @@ #define MAX_BPFTOOL_CMD_LEN (256) +int detect_bpftool_path(char *buffer, size_t size); int run_bpftool_command(char *args); int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_len); diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c new file mode 100644 index 000000000000..e9724c55d1dc --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include "bpftool_helpers.h" +#include "bpftool_ringbuf.skel.h" + +#define WAIT_STEPS 500 +#define WAIT_US 10000 + +struct consumer { + pid_t pid; + int fd; +}; + +static void consumer_cleanup(struct consumer *child) +{ + if (child->pid > 0) { + kill(child->pid, SIGKILL); + while (waitpid(child->pid, NULL, 0) < 0 && errno == EINTR) + ; + child->pid = -1; + } + if (child->fd >= 0) { + close(child->fd); + child->fd = -1; + } +} + +static bool consumer_start(struct consumer *child, int map_fd, const char *format, + const char *pin_path, const char *option, + bool pair, bool capture_errors) +{ + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + char path[PATH_MAX]; + char *argv[12], id[16]; + int out[2], ready[2], n = 0, err, err_fd; + struct pollfd pfd; + + if (!ASSERT_OK(detect_bpftool_path(path, sizeof(path)), "bpftool path") || + !ASSERT_OK(bpf_map_get_info_by_fd(map_fd, &info, &len), "map info")) + return false; + snprintf(id, sizeof(id), "%u", info.id); + argv[n++] = path; + if (format) + argv[n++] = (char *)format; + argv[n++] = "map"; + argv[n++] = "event_pipe"; + argv[n++] = pin_path ? "pinned" : "id"; + argv[n++] = pin_path ? (char *)pin_path : id; + if (option) { + argv[n++] = (char *)option; + argv[n++] = "0"; + if (pair) { + argv[n++] = "index"; + argv[n++] = "0"; + } + } + argv[n] = NULL; + if (!ASSERT_OK(pipe2(out, O_CLOEXEC), "output pipe")) + return false; + if (!ASSERT_OK(pipe2(ready, O_CLOEXEC), "exec pipe")) { + close(out[0]); + close(out[1]); + return false; + } + child->pid = fork(); + if (!child->pid) { + close(out[0]); + close(ready[0]); + err_fd = capture_errors ? out[1] : open("/dev/null", O_WRONLY); + if (dup2(out[1], STDOUT_FILENO) < 0 || + dup2(err_fd, STDERR_FILENO) < 0) + goto exec_fail; + if (!capture_errors) + close(err_fd); + close(out[1]); + execv(path, argv); +exec_fail: + err = errno; + if (write(ready[1], &err, sizeof(err)) != sizeof(err)) + _exit(126); + _exit(127); + } + close(out[1]); + close(ready[1]); + child->fd = out[0]; + pfd = (struct pollfd) { .fd = ready[0], .events = POLLIN }; + /* EOF on the close-on-exec pipe distinguishes exec from inherited handlers. */ + err = child->pid > 0 ? poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000) : -1; + if (!ASSERT_GT(child->pid, 0, "fork") || + !ASSERT_GT(err, 0, "exec timeout") || + !ASSERT_EQ(read(ready[0], &err, sizeof(err)), 0, "exec")) { + close(ready[0]); + consumer_cleanup(child); + return false; + } + close(ready[0]); + return true; +} + +static bool consumer_finish(struct consumer *child, int signo, bool success, + char *output, size_t size) +{ + int status = 0, i; + pid_t ret = 0; + ssize_t n; + size_t used = 0; + + if (!ASSERT_OK(fcntl(child->fd, F_SETFL, O_NONBLOCK), "nonblocking output")) + return false; + if (signo && !ASSERT_OK(kill(child->pid, signo), "signal consumer")) + return false; + for (i = 0; i < WAIT_STEPS; i++) { + while (used < size - 1 && + (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + ret = waitpid(child->pid, &status, WNOHANG); + if (ret == child->pid) + break; + if (ret < 0 && errno != EINTR) + break; + usleep(WAIT_US); + } + if (!ASSERT_EQ(ret, child->pid, "bounded consumer exit")) + return false; + child->pid = -1; + while (used < size - 1 && (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + output[used] = '\0'; + return ASSERT_TRUE(WIFEXITED(status), "normal exit") && + ASSERT_EQ(WEXITSTATUS(status) == 0, success, "exit status"); +} + +static bool consumer_ready(struct consumer *child) +{ + unsigned long long caught; + char path[64], line[256]; + int i; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", child->pid); + for (i = 0; i < WAIT_STEPS; i++) { + f = fopen(path, "r"); + if (!f) + break; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "SigCgt: %llx", &caught) == 1 && + (caught & (1ULL << (SIGINT - 1))) && + (caught & (1ULL << (SIGTERM - 1)))) { + fclose(f); + return true; + } + } + fclose(f); + usleep(WAIT_US); + } + return ASSERT_TRUE(false, "consumer signal handlers ready"); +} + +static bool emit_record(struct bpftool_ringbuf *skel, int record) +{ + char packet[64] = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = packet, + .data_size_in = sizeof(packet), + ); + + skel->bss->record = record; + return ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.produce), + &opts), "produce record") && + ASSERT_OK(skel->bss->output_err, "ringbuf output"); +} + +static bool consumed(unsigned long *position, unsigned long expected) +{ + int i; + + for (i = 0; i < WAIT_STEPS; i++) { + if (__atomic_load_n(position, __ATOMIC_ACQUIRE) == expected) + return true; + usleep(WAIT_US); + } + return ASSERT_EQ(*position, expected, "consumer position"); +} + +static void check_json(char *output, const char *expected) +{ + char *src = output, *dst = output; + bool quoted = false, escaped = false; + + /* Ignore formatting whitespace while checking the entire JSON document. */ + while (*src) { + if (quoted || !isspace((unsigned char)*src)) + *dst++ = *src; + if (!escaped && *src == '"') + quoted = !quoted; + escaped = !escaped && quoted && *src == '\\'; + src++; + } + *dst = '\0'; + ASSERT_STREQ(output, expected, "JSON records"); +} + +static void test_consumer(const char *format, bool idle, int signo, bool pinned) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + struct bpftool_ringbuf *skel; + unsigned long *position = MAP_FAILED; + int page_size = getpagesize(), fd; + char pin_dir[] = "/sys/fs/bpf/bpftool_ringbuf_XXXXXX"; + char pin_path[sizeof(pin_dir) + sizeof("/map")]; + bool dir_created = false, map_pinned = false; + char output[4096]; + struct pollfd pfd; + + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, page_size); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load")) + goto out; + fd = bpf_map__fd(skel->maps.ringbuf); + if (pinned) { + if (!ASSERT_OK_PTR(mkdtemp(pin_dir), "create pin directory")) + goto out; + dir_created = true; + snprintf(pin_path, sizeof(pin_path), "%s/map", pin_dir); + if (!ASSERT_OK(bpf_obj_pin(fd, pin_path), "pin ringbuf")) + goto out; + map_pinned = true; + } + position = mmap(NULL, page_size, PROT_READ, MAP_SHARED, fd, 0); + if (!ASSERT_NEQ(position, MAP_FAILED, "consumer mmap")) + goto out; + if (!idle && (!emit_record(skel, 0) || !emit_record(skel, 1))) + goto out; + if (!consumer_start(&child, fd, format, pinned ? pin_path : NULL, + NULL, false, false) || + !consumer_ready(&child)) + goto out; + if (!idle) { + /* Both prefilled records occupy 16 bytes including their headers. */ + if (!consumed(position, 32)) + goto out; + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "records flushed before exit") || + !ASSERT_TRUE(pfd.revents & POLLIN, "record output readable") || + !emit_record(skel, 2) || !consumed(position, 64)) + goto out; + } + if (!consumer_finish(&child, signo, true, output, sizeof(output))) + goto out; + if (format && !strcmp(format, "-p")) + ASSERT_STREQ(output, idle ? "[]\n" : + "[{\n" + " \"size\": 2,\n" + " \"data\": [0,255\n" + " ]\n" + " },{\n" + " \"size\": 5,\n" + " \"data\": [1,2,3,4,5\n" + " ]\n" + " },{\n" + " \"size\": 17,\n" + " \"data\": [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32\n" + " ]\n" + " }\n" + "]\n", "pretty JSON records"); + else if (format) + check_json(output, idle ? "[]" : + "[{\"size\":2,\"data\":[0,255]}," + "{\"size\":5,\"data\":[1,2,3,4,5]}," + "{\"size\":17,\"data\":[16,17,18,19,20,21,22,23," + "24,25,26,27,28,29,30,31,32]}]"); + else + ASSERT_STREQ(output, idle ? "" : + "== size: 2 =====\n00 ff\n" + "== size: 5 =====\n01 02 03 04 05\n" + "== size: 17 =====\n10 11 12 13 14 15 16 17 " + "18 19 1a 1b 1c 1d 1e 1f\n20\n", "plain records"); +out: + consumer_cleanup(&child); + if (position != MAP_FAILED) + munmap(position, page_size); + if (map_pinned) + ASSERT_OK(unlink(pin_path), "unpin ringbuf"); + if (dir_created) + ASSERT_OK(rmdir(pin_dir), "remove pin directory"); + bpftool_ringbuf__destroy(skel); +} + +static void test_perf_consumer(void) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + unsigned long long seconds, nanoseconds; + struct bpftool_ringbuf *skel; + char output[16384], expected[16384]; + int nr_cpus, cpu, index, offset = 0, i, used = 0, fields; + struct pollfd pfd; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "possible cpus")) + return; + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize()); + bpf_map__set_max_entries(skel->maps.perfbuf, nr_cpus); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load") || + !consumer_start(&child, bpf_map__fd(skel->maps.perfbuf), NULL, + NULL, NULL, false, false) || !consumer_ready(&child) || + !emit_record(skel, 3)) + goto out; + /* One large record flushes the existing buffered perf output callback. */ + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "perf record output") || + !ASSERT_TRUE(pfd.revents & POLLIN, "perf output readable") || + !consumer_finish(&child, SIGINT, true, output, sizeof(output))) + goto out; + fields = sscanf(output, "== @%llu.%llu CPU: %d index: %d =====\n%n", + &seconds, &nanoseconds, &cpu, &index, &offset); + if (!ASSERT_EQ(fields, 4, "perf header") || + !ASSERT_GT(offset, 0, "perf payload offset")) + goto out; + ASSERT_GT(seconds * 1000000000ULL + nanoseconds, 0, "perf timestamp"); + ASSERT_LT(nanoseconds, 1000000000ULL, "perf timestamp nanoseconds"); + ASSERT_GE(cpu, 0, "perf cpu"); + ASSERT_LT(cpu, nr_cpus, "perf cpu range"); + ASSERT_EQ(index, cpu, "perf index"); + for (i = 0; i < sizeof(skel->rodata->perf_payload); i++) { + const char *separator = !i ? "" : !(i % 16) ? "\n" : + !(i % 8) ? " " : " "; + + used += snprintf(expected + used, sizeof(expected) - used, + "%s%02x", separator, i == 1 ? 0xff : 0); + } + snprintf(expected + used, sizeof(expected) - used, "\n"); + ASSERT_STREQ(output + offset, expected, "perf payload"); +out: + consumer_cleanup(&child); + bpftool_ringbuf__destroy(skel); +} + +static void test_reject(enum bpf_map_type type, const char *option, bool pair, bool json) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + bool ring = type == BPF_MAP_TYPE_RINGBUF || type == BPF_MAP_TYPE_USER_RINGBUF; + const char *expected = option && !strcmp(option, "foobar") ? + "{\"error\":\"what is 'foobar'?\"}" : option ? + "{\"error\":\"ring buffer maps do not support cpu or index arguments\"}" : + "{\"error\":\"map is not a perf event array or ring buffer\"}"; + char output[4096]; + int fd; + + fd = bpf_map_create(type, NULL, ring ? 0 : 4, ring ? 0 : 4, + ring ? getpagesize() : 1, NULL); + if (!ASSERT_GE(fd, 0, "create map")) + return; + if (consumer_start(&child, fd, json ? "-j" : NULL, NULL, option, pair, true) && + consumer_finish(&child, 0, false, output, sizeof(output))) { + if (json) + check_json(output, expected); + else + ASSERT_GT(strlen(output), 0, "error diagnostic"); + } + consumer_cleanup(&child); + close(fd); +} + +void test_bpftool_ringbuf(void) +{ + if (test__start_subtest("perf_event_array")) + test_perf_consumer(); + if (test__start_subtest("plain")) + test_consumer(NULL, false, SIGINT, false); + if (test__start_subtest("json")) + test_consumer("-j", false, SIGTERM, false); + if (test__start_subtest("pretty_json")) + test_consumer("-p", false, SIGINT, false); + if (test__start_subtest("pinned_plain")) + test_consumer(NULL, false, SIGTERM, true); + if (test__start_subtest("pinned_json")) + test_consumer("-j", false, SIGINT, true); + if (test__start_subtest("pinned_pretty_json")) + test_consumer("-p", false, SIGTERM, true); + if (test__start_subtest("idle_sigint")) + test_consumer(NULL, true, SIGINT, false); + if (test__start_subtest("idle_sigterm_json")) + test_consumer("-j", true, SIGTERM, false); + if (test__start_subtest("idle_sigint_pretty_json")) + test_consumer("-p", true, SIGINT, false); + if (test__start_subtest("reject_array")) + test_reject(BPF_MAP_TYPE_ARRAY, NULL, false, false); + if (test__start_subtest("reject_user_ringbuf")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, false); + if (test__start_subtest("reject_cpu")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", false, false); + if (test__start_subtest("reject_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "index", false, false); + if (test__start_subtest("reject_cpu_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, false); + if (test__start_subtest("reject_user_ringbuf_json")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, true); + if (test__start_subtest("reject_unknown_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "foobar", false, true); + if (test__start_subtest("reject_cpu_index_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, true); +} diff --git a/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c new file mode 100644 index 000000000000..bbbc93c30a83 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} perfbuf SEC(".maps"); + +/* + * Include the u32 raw size in perf's 8-byte alignment to avoid padding. + * Keep plain output large enough to flush stdio before the consumer exits. + */ +const unsigned char perf_payload[4092] = { 0x00, 0xff }; + +int record; +int output_err; + +SEC("socket") +int produce(struct __sk_buff *skb) +{ + unsigned char first[] = { 0x00, 0xff }; + unsigned char second[] = { 1, 2, 3, 4, 5 }; + unsigned char third[] = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + }; + + if (record == 0) + output_err = bpf_ringbuf_output(&ringbuf, first, sizeof(first), 0); + else if (record == 1) + output_err = bpf_ringbuf_output(&ringbuf, second, sizeof(second), 0); + else if (record == 2) + output_err = bpf_ringbuf_output(&ringbuf, third, sizeof(third), 0); + else + output_err = bpf_perf_event_output(skb, &perfbuf, BPF_F_CURRENT_CPU, + (void *)perf_payload, sizeof(perf_payload)); + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; From 79dc258c9392051420a26f1504c647bd3d27c66a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 18 Sep 2026 12:26:10 +0200 Subject: [PATCH 210/660] selftests/bpf: Replace %pK output with 0 Networking used to print the socket pointer with %p as output in the /proc interface. This was later changed to %pK in order not to reveal the actual pointer value. The output format has then been copied to bpf tests which produce the same format including the %pK format modifier. Networking recently replaced the socket output with a plain 0 because the socket pointer added no value to the output and the %pK should be removed from library handling. The format remained otherwise unchanged in order not to break any tools which parsing this information. This was done via 7c0ec6288b49 ("net: Replace %pK output with 0"). This change removes the %pK modifier, following the change in networking. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260918102610.gxA3km8_@linutronix.de --- tools/testing/selftests/bpf/README.rst | 2 +- tools/testing/selftests/bpf/prog_tests/snprintf.c | 4 ++-- tools/testing/selftests/bpf/progs/bpf_iter_netlink.c | 4 ++-- tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c | 11 +++++------ tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c | 11 +++++------ tools/testing/selftests/bpf/progs/bpf_iter_udp4.c | 4 ++-- tools/testing/selftests/bpf/progs/bpf_iter_udp6.c | 4 ++-- tools/testing/selftests/bpf/progs/bpf_iter_unix.c | 5 ++--- tools/testing/selftests/bpf/progs/test_snprintf.c | 4 ++-- 9 files changed, 23 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 07c834433b38..75ff1142c185 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -187,7 +187,7 @@ The symptom for ``bpf_iter/netlink`` looks like ... 15: (bf) r2 = r7 16: (0f) r2 += r1 - ; BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + ; BPF_SEQ_PRINTF(seq, "%p %-3d ", s, s->sk_protocol); 17: (7b) *(u64 *)(r7 +0) = r2 only read is supported diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 4e4a82d54f79..ccfb02bd4e7c 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c @@ -15,8 +15,8 @@ #define EXP_SYM_OUT "schedule schedule+0x0/" #define MIN_SYM_RET sizeof(EXP_SYM_OUT) -/* The third specifier, %p, is a hashed pointer which changes on every reboot */ -#define EXP_ADDR_OUT "0000000000000000 ffff00000add4e55 " +/* The second specifier, %p, is a hashed pointer which changes on every reboot */ +#define EXP_ADDR_OUT "ffff00000add4e55 " #define EXP_ADDR_RET sizeof(EXP_ADDR_OUT "unknownhashedptr") #define EXP_STR_OUT "str1 a b c d e longstr" diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c index 00b2ceae81fb..b9107d3ab603 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c @@ -25,12 +25,12 @@ int dump_netlink(struct bpf_iter__netlink *ctx) return 0; if (ctx->meta->seq_num == 0) - BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " + BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " "Rmem Wmem Dump Locks Drops " "Inode\n"); s = &nlk->sk; - BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + BPF_SEQ_PRINTF(seq, "0 %-3d ", s->sk_protocol); if (!nlk->groups) { group = 0; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c index b1e509b231cd..d149cf0d07be 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c @@ -131,8 +131,7 @@ static int dump_tcp_sock(struct seq_file *seq, struct tcp_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -161,10 +160,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ", seq_num, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -185,9 +184,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, seq_num, irsk->ir_loc_addr, irsk->ir_num, irsk->ir_rmt_addr, bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c index dbc7166aee91..cd65fa81f121 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c @@ -135,8 +135,7 @@ static int dump_tcp6_sock(struct seq_file *seq, struct tcp6_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -170,10 +169,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -201,9 +200,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c index 23b2aa2604de..d4079a619da6 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c @@ -57,13 +57,13 @@ int dump_udp4(struct bpf_iter__udp *ctx) BPF_SEQ_PRINTF(seq, "%5d: %08X:%04X %08X:%04X ", ctx->bucket, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c index c48b05aa2a4b..dadc5f73ff09 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c @@ -65,13 +65,13 @@ int dump_udp6(struct bpf_iter__udp *ctx) dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter); return 0; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index a2652c8c3616..df3d971b21e8 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c @@ -42,10 +42,9 @@ int dump_unix(struct bpf_iter__unix *ctx) seq = ctx->meta->seq; seq_num = ctx->meta->seq_num; if (seq_num == 0) - BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); + BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); - BPF_SEQ_PRINTF(seq, "%pK: %08X %08X %08X %04X %02X %8lu", - unix_sk, + BPF_SEQ_PRINTF(seq, "%08u: %08X %08X %08X %04X %02X %8lu", 0, sk->sk_refcnt.refs.counter, 0, sk->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0, diff --git a/tools/testing/selftests/bpf/progs/test_snprintf.c b/tools/testing/selftests/bpf/progs/test_snprintf.c index 8fda07544023..48a0025cf2ca 100644 --- a/tools/testing/selftests/bpf/progs/test_snprintf.c +++ b/tools/testing/selftests/bpf/progs/test_snprintf.c @@ -57,8 +57,8 @@ int handler(const void *ctx) sym_ret = BPF_SNPRINTF(sym_out, sizeof(sym_out), "%ps %pS %pB", &schedule, &schedule, &schedule); /* Kernel pointers */ - addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%pK %px %p", - 0, 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); + addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%px %p", + 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); /* Strings and single-byte character embedding */ str_ret = BPF_SNPRINTF(str_out, sizeof(str_out), "%s % 9c %+2c %-3c %04c %0c %+05s", str1, 'a', 'b', 'c', 'd', 'e', longstr); From 63b13537e6b223699dabb60ec0009b0c976e7181 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Tue, 22 Sep 2026 10:58:02 -0700 Subject: [PATCH 211/660] bpf: Speed up htab lookups for u32/u64 keys Four- and eight-byte keys are common enough to warrant specialized htab lookups. Use jhash_1word() and jhash_2words() for these key sizes instead of the generic jhash2() path. For JITed BPF_MAP_TYPE_HASH lookups, htab_map_gen_lookup() also knows the map's fixed key size. Select u32- and u64-specific lookup entry points so the compiler can specialize both hashing and key comparison. Other key sizes continue to use the generic lookup entry point. This preserves the jhash2() result and lookup semantics while avoiding generic length handling. Measure this with six runs of: ./bench -w3 -d10 -a bpf-hashmap-lookup \ --key_size KEY_SIZE --max_entries MAX_ENTRIES \ --nr_entries NR_ENTRIES --nr_loops NR_LOOPS \ --map_flags 0x40 inside a vng guest with two vCPUs and 4 GiB of RAM. Run the baseline first and the optimized kernel second. Use these workloads: max_entries nr_entries nr_loops small 512 256 8,388,608 medium 10,000 5,000 8,000,000 large 100,000 50,000 8,000,000 Mean throughput in million lookups per second is: Baseline: key size (bytes) 1 4 8 10 small 89.45 91.21 103.40 91.74 medium 125.97 80.25 91.94 78.52 large 140.67 53.56 58.68 47.31 Optimized: key size (bytes) 1 4 8 10 small 86.95 197.77 168.65 90.88 medium 124.97 147.48 137.76 77.51 large 131.06 78.51 76.04 47.25 Change: key size (bytes) 1 4 8 10 small -2.79% +116.83% +63.11% -0.94% medium -0.79% +83.78% +49.83% -1.28% large -6.84% +46.58% +29.58% -0.12% The u32 specialization improves lookup throughput by 47-117%, and the u64 specialization improves it by 30-63%. Ten-byte controls remain within a 1.3% slowdown. One-byte controls range from -6.8% to -0.8%. Signed-off-by: Mykyta Yatsenko Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Link: https://patch.msgid.link/20260922-hashtab_fast_hashfn-v3-1-b7b0e9bc33ac@meta.com --- kernel/bpf/hashtab.c | 63 +++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6f331c80130d..f744a42bb813 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -677,11 +677,18 @@ free_htab: return ERR_PTR(err); } -static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) +static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) { - if (likely(key_len % 4 == 0)) + const u32 *k = key; + + if (key_len == sizeof(u32)) + return jhash_1word(k[0], hashrnd); + else if (key_len == sizeof(u64)) + return jhash_2words(k[0], k[1], hashrnd); + else if (likely(key_len % 4 == 0)) return jhash2(key, key_len / 4, hashrnd); - return jhash(key, key_len, hashrnd); + else + return jhash(key, key_len, hashrnd); } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) @@ -712,9 +719,9 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, + u32 hash, void *key, + u32 key_size, u32 n_buckets) { struct hlist_nulls_node *n; struct htab_elem *l; @@ -730,22 +737,15 @@ again: return NULL; } -/* Called from syscall or from eBPF program directly, so - * arguments have to match bpf_map_lookup_elem() exactly. - * The return value is adjusted by BPF instructions - * in htab_map_gen_lookup(). - */ -static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 key_size) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; struct htab_elem *l; - u32 hash, key_size; + u32 hash; WARN_ON_ONCE(!bpf_rcu_lock_held()); - key_size = map->key_size; - hash = htab_map_hash(key, key_size, htab->hashrnd); head = select_bucket(htab, hash); @@ -755,6 +755,28 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) return l; } +/* + * Called from syscall or from eBPF program directly, so + * arguments have to match bpf_map_lookup_elem() exactly. + * The return value is adjusted by BPF instructions + * in htab_map_gen_lookup(). u32/u64 specializations + * allow utilizing compile-time key_size for optimization. + */ +static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, map->key_size); +} + +static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u32)); +} + +static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u64)); +} + static void *htab_map_lookup_elem(struct bpf_map *map, void *key) { struct htab_elem *l = __htab_map_lookup_elem(map, key); @@ -783,7 +805,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32, + (void *(*)(struct bpf_map *map, void *key))NULL)); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64, + (void *(*)(struct bpf_map *map, void *key))NULL)); + if (map->key_size == sizeof(u32)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32); + else if (map->key_size == sizeof(u64)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64); + else + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + From 1832696b22078f57069a676266caaeb4b0562ad7 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 22 Sep 2026 13:00:50 -0700 Subject: [PATCH 212/660] bpf: Add bpf_call_rcu() kfunc BPF programs that manage their own objects have no way to run their own logic once an RCU grace period has elapsed. bpf_obj_drop() defers a free, but returning an index to an allocator or unpinning a resource once readers are done has no equivalent. sched_ext's BPF library works around this today by pushing freed nodes onto a list and having a userspace thread call membarrier(MEMBARRIER_CMD_GLOBAL) and then run a BPF program to reclaim them. Add: int bpf_call_rcu(struct bpf_rcu_head *rh, void *map, int (*callback)(struct bpf_map *map, void *key, void *value)); @rh is a struct bpf_rcu_head embedded in a value of @map, so the callback runs as callback(map, key, value) for the element it lives in and needs no cookie. A head can only be armed once, which bounds outstanding work by the number of elements. struct bpf_rcu_head holds the callback state inline rather than a pointer to it, as bpf_timer, bpf_wq and bpf_task_work do, because there is nothing to cancel and so nothing that has to outlive the map value. That avoids an allocation and a state machine on the arming path at the cost of 48 bytes per element. An RCU callback cannot be cancelled, so everything it touches has to stay alive until it runs: - The callback is the program's text, so arming takes a program reference as bpf_timer, bpf_wq and bpf_task_work do, dropped once the callback returns. bpf_prog_inc_not_zero() also fails the arm with -EBADF once the program is dying. - The map is held by that reference through used_maps. An inner map is not, so bpf_rcu_head is rejected in one. - The field is only accepted in BPF_MAP_TYPE_ARRAY, whose elements are never freed individually. A hash element can be deleted and recycled while a callback is queued on it. - The head is disarmed before the callback runs so it can be armed again from there, which takes a new program reference before the running callback drops its own. Arming therefore fails with -EPERM once the map is held by neither a process nor bpffs. bpf_iter hands a program a writable pointer to the live element, which would let it overwrite a queued head, so bpf_iter_attach_map() rejects maps carrying one. The callback is verified non-sleepable even when the caller is sleepable, and RCU invokes it with BH disabled. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922200208.3203834-2-puranjay@kernel.org --- include/linux/bpf.h | 10 +++++ include/uapi/linux/bpf.h | 4 ++ kernel/bpf/btf.c | 7 +++ kernel/bpf/helpers.c | 75 +++++++++++++++++++++++++++++++ kernel/bpf/map_in_map.c | 4 ++ kernel/bpf/map_iter.c | 6 +++ kernel/bpf/syscall.c | 11 ++++- kernel/bpf/verifier.c | 81 +++++++++++++++++++++++++++++++++- tools/include/uapi/linux/bpf.h | 4 ++ 9 files changed, 199 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index fd22db8bc6c5..e7c5e203eddd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -215,6 +215,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_RCU_HEAD = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -269,6 +270,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int rcu_head_off; struct btf_field fields[]; }; @@ -374,6 +376,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_RCU_HEAD: + return "bpf_rcu_head"; default: WARN_ON_ONCE(1); return "unknown"; @@ -414,6 +418,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_RCU_HEAD: + return sizeof(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -448,6 +454,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_RCU_HEAD: + return __alignof__(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -480,6 +488,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -925,6 +934,7 @@ enum bpf_arg_type { ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_RCU_HEAD, /* pointer to bpf_rcu_head */ ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 6330b7d745c5..0aaa54359aeb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -7611,6 +7611,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8))); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 314ecb0e593b..4a1fa4fbdf4e 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3696,6 +3696,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_TIMER, "bpf_timer", true }, { BPF_WORKQUEUE, "bpf_wq", true }, { BPF_TASK_WORK, "bpf_task_work", true }, + { BPF_RCU_HEAD, "bpf_rcu_head", true }, { BPF_LIST_HEAD, "bpf_list_head", false }, { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, @@ -3881,6 +3882,7 @@ static int btf_find_field_one(const struct btf *btf, case BPF_RB_NODE: case BPF_REFCOUNT: case BPF_TASK_WORK: + case BPF_RCU_HEAD: ret = btf_find_struct(btf, var_type, off, sz, field_type, info_cnt ? &info[0] : &tmp); if (ret < 0) @@ -4176,6 +4178,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->wq_off = -EINVAL; rec->refcount_off = -EINVAL; rec->task_work_off = -EINVAL; + rec->rcu_head_off = -EINVAL; for (i = 0; i < cnt; i++) { field_type_size = btf_field_type_size(info_arr[i].type); if (info_arr[i].off + field_type_size > value_size) { @@ -4219,6 +4222,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type WARN_ON_ONCE(rec->task_work_off >= 0); rec->task_work_off = rec->fields[i].offset; break; + case BPF_RCU_HEAD: + WARN_ON_ONCE(rec->rcu_head_off >= 0); + rec->rcu_head_off = rec->fields[i].offset; + break; case BPF_REFCOUNT: WARN_ON_ONCE(rec->refcount_off >= 0); /* Cache offset for faster lookup at runtime */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 501c7ce35cba..8a01dd4058a0 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4805,6 +4805,80 @@ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct b return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } +typedef int (*bpf_rcu_callback_t)(struct bpf_map *map, void *key, void *value); + +/* Actual type for struct bpf_rcu_head */ +struct bpf_rcu_head_kern { + struct rcu_head rcu; + bpf_callback_t callback_fn; + struct bpf_map *map; + struct bpf_prog *prog; + u32 armed; +} __aligned(8); + +static void bpf_rcu_run_callback(struct rcu_head *rcu) +{ + struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); + bpf_callback_t callback_fn = rh->callback_fn; + struct bpf_prog *prog = rh->prog; + struct bpf_map *map = rh->map; + void *value, *key; + u32 idx; + + value = (void *)rh - map->record->rcu_head_off; + key = map_key_from_value(map, value, &idx); + + /* Pairs with the arming cmpxchg(): rh may be re-armed as soon as this store lands. */ + smp_store_release(&rh->armed, 0); + + rcu_read_lock_dont_migrate(); + callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + rcu_read_unlock_migrate(); + + bpf_prog_put(prog); +} + +/** + * bpf_call_rcu - Invoke a BPF callback after an RCU grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + struct bpf_rcu_head_kern *rhk = (void *)rh; + struct bpf_map *map = map__const_map; + struct bpf_prog *prog; + + BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head)); + BUILD_BUG_ON(__alignof__(struct bpf_rcu_head_kern) != __alignof__(struct bpf_rcu_head)); + BTF_TYPE_EMIT(struct bpf_rcu_head); + + /* A queued callback cannot be cancelled, so a self-rearming one would pin prog and map. */ + if (!atomic64_read(&map->usercnt)) + return -EPERM; + + if (cmpxchg(&rhk->armed, 0, 1)) + return -EBUSY; + + prog = bpf_prog_inc_not_zero(aux->prog); + if (IS_ERR(prog)) { + WRITE_ONCE(rhk->armed, 0); + return -EBADF; + } + + rhk->callback_fn = (bpf_callback_t)(void *)callback; + rhk->map = map; + rhk->prog = prog; + call_rcu(&rhk->rcu, bpf_rcu_run_callback); + return 0; +} + static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, struct bpf_dynptr_kern *ptr) { @@ -5100,6 +5174,7 @@ BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async) diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4bdf64..5ee8aae7435b 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -25,6 +25,10 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); + /* An inner map has no used_maps reference to hold it under a queued callback. */ + if (btf_record_has_field(inner_map->record, BPF_RCU_HEAD)) + return ERR_PTR(-EOPNOTSUPP); + inner_map_meta_size = sizeof(*inner_map_meta); /* In some cases verifier needs to access beyond just base map. */ if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index c19b360bad9e..2077d46d167c 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -117,6 +117,12 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, goto put_map; } + /* The value ctx arg is writable and aliases the live element. */ + if (btf_record_has_field(map->record, BPF_RCU_HEAD)) { + err = -EOPNOTSUPP; + goto put_map; + } + if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 113486b15d29..74496fd716d3 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -687,6 +687,7 @@ void btf_record_free(struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to release */ break; default: @@ -741,6 +742,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to acquire */ break; default: @@ -874,6 +876,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) case BPF_LIST_NODE: case BPF_RB_NODE: case BPF_REFCOUNT: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -1280,7 +1283,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, map->record = btf_parse_fields(btf, value_type, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR | - BPF_TASK_WORK, + BPF_TASK_WORK | BPF_RCU_HEAD, map->value_size); if (!IS_ERR_OR_NULL(map->record)) { int i; @@ -1322,6 +1325,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, goto free_map_tab; } break; + case BPF_RCU_HEAD: + if (map->map_type != BPF_MAP_TYPE_ARRAY) { + ret = -EOPNOTSUPP; + goto free_map_tab; + } + break; case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d62c0f74cff5..64db47964ff9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -535,6 +535,7 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); +static bool is_call_rcu_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); @@ -577,6 +578,10 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; + /* bpf_call_rcu callbacks are never sleepable. */ + if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) + return false; + /* bpf_wq and bpf_task_work callbacks are always sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) @@ -7590,6 +7595,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ case BPF_TASK_WORK: field_off = map->record->task_work_off; break; + case BPF_RCU_HEAD: + field_off = map->record->rcu_head_off; + break; case BPF_WORKQUEUE: field_off = map->record->wq_off; break; @@ -8448,6 +8456,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, [ARG_PTR_TO_WORKQUEUE] = &map_value_types, [ARG_PTR_TO_TASK_WORK] = &map_value_types, + [ARG_PTR_TO_RCU_HEAD] = &map_value_types, [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, [ARG_PTR_TO_ARENA] = &arena_types, [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, @@ -8894,6 +8903,8 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat obj_name = "timer"; else if (rec->task_work_off >= 0) obj_name = "bpf_task_work"; + else if (rec->rcu_head_off >= 0) + obj_name = "bpf_rcu_head"; verbose(env, "%s pointer in %s map_uid=%d ", obj_name, reg_arg_name(env, obj_argno), meta->map.uid); @@ -9442,6 +9453,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err < 0) return err; break; + case ARG_PTR_TO_RCU_HEAD: + err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map); + if (err < 0) + return err; + break; case ARG_PTR_TO_IRQ_FLAG: err = process_irq_flag(env, reg, argno, meta); if (err < 0) @@ -10955,6 +10971,40 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_rcu_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr; + u32 map_uid = caller->regs[BPF_REG_2].map_uid; + + /* + * callback_fn(struct bpf_map *map, void *key, void *value); + */ + callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; + + callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; + + callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; + __mark_reg_known_zero(&callee->regs[BPF_REG_3]); + callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; + + /* unused */ + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_async_callback_fn = true; + callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); + return 0; +} + static bool is_rbtree_lock_required_kfunc(u32 btf_id); static void account_processed_insn(struct bpf_verifier_env *env) @@ -12216,7 +12266,8 @@ enum { KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, KF_ARG_PROG_AUX_ID, - KF_ARG_TIMER_ID + KF_ARG_TIMER_ID, + KF_ARG_RCU_HEAD_ID }; BTF_ID_LIST(kf_arg_btf_ids) @@ -12230,6 +12281,7 @@ BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) BTF_ID(struct, bpf_prog_aux) BTF_ID(struct, bpf_timer) +BTF_ID(struct, bpf_rcu_head) static bool __is_kfunc_ptr_arg_type(const struct btf *btf, const struct btf_param *arg, int type) @@ -12288,6 +12340,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); } +static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg) +{ + return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID); +} + static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg) { return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); @@ -12571,6 +12628,7 @@ enum special_kfunc_type { KF___bpf_trap, KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, + KF_bpf_call_rcu, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_arena_reserve_pages, @@ -12664,6 +12722,7 @@ BTF_ID(func, bpf_dynptr_file_discard) BTF_ID(func, __bpf_trap) BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) +BTF_ID(func, bpf_call_rcu) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) BTF_ID(func, bpf_arena_reserve_pages) @@ -12735,6 +12794,11 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; } +static bool is_call_rcu_kfunc(u32 func_id) +{ + return func_id == special_kfunc_list[KF_bpf_call_rcu]; +} + static bool is_task_work_add_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || @@ -12941,6 +13005,8 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, arg_type = ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) arg_type = ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_RCU_HEAD; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) @@ -13442,7 +13508,8 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id) static bool is_async_callback_calling_kfunc(u32 btf_id) { return is_bpf_wq_set_callback_kfunc(btf_id) || - is_task_work_add_kfunc(btf_id); + is_task_work_add_kfunc(btf_id) || + is_call_rcu_kfunc(btf_id); } bool bpf_is_throw_kfunc(struct bpf_insn *insn) @@ -14259,6 +14326,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + if (is_call_rcu_kfunc(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_rcu_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6330b7d745c5..0aaa54359aeb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -7611,6 +7611,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8))); From 908a60853b8dfe3bc9ad33598330bdcd03489b7c Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 22 Sep 2026 13:00:51 -0700 Subject: [PATCH 213/660] selftests/bpf: Add tests for bpf_call_rcu() Cover the callback running after a grace period with the right map, key and value, -EBUSY on a second arm, reuse of the head once disarmed, a callback arming itself again, and teardown with a callback queued. struct bpf_rcu_head is not the first member of the map value, so the callback's recovery of the value from the head is exercised. arm() wraps both arms in an RCU read section, otherwise a grace period may elapse between them and the second one legitimately succeeds. Negative tests: a hash map created with the same BTF, the map used as an inner map, and an iterator attach, all checked for -EOPNOTSUPP; plus verifier rejection of a mismatched map, a map with no bpf_rcu_head, a head at the wrong offset, a head on the stack, and a sleepable callback. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922200208.3203834-3-puranjay@kernel.org --- .../selftests/bpf/prog_tests/call_rcu.c | 271 ++++++++++++++++++ tools/testing/selftests/bpf/progs/call_rcu.c | 93 ++++++ .../selftests/bpf/progs/call_rcu_fail.c | 114 ++++++++ 3 files changed, 478 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/call_rcu.c create mode 100644 tools/testing/selftests/bpf/progs/call_rcu.c create mode 100644 tools/testing/selftests/bpf/progs/call_rcu_fail.c diff --git a/tools/testing/selftests/bpf/prog_tests/call_rcu.c b/tools/testing/selftests/bpf/prog_tests/call_rcu.c new file mode 100644 index 000000000000..bfa6512b9052 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/call_rcu.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "call_rcu.skel.h" +#include "call_rcu_fail.skel.h" + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +/* + * Force the grace period, then poll: the callback still has to be invoked + * afterwards, and call_rcu() is lazy on a CONFIG_RCU_LAZY kernel. + */ +static bool wait_for_callbacks(struct call_rcu *skel, int expected) +{ + int i, got = 0; + + kern_sync_rcu(); + for (i = 0; i < 3000; i++) { + got = __atomic_load_n(&skel->bss->callbacks, __ATOMIC_ACQUIRE); + if (got >= expected) + return true; + usleep(10000); + } + return ASSERT_GE(got, expected, "callbacks"); +} + +static void test_call_rcu_run(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + struct elem elem; + __u32 key = 1; + int err; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"); + ASSERT_EQ(skel->bss->busy_err, -EBUSY, "busy_err"); + + if (!wait_for_callbacks(skel, 1)) + goto out; + + ASSERT_EQ(skel->bss->cb_key, key, "cb_key"); + ASSERT_EQ(skel->bss->cb_val, 0xdeadbeef, "cb_val"); + ASSERT_EQ(skel->bss->cb_max_entries, bpf_map__max_entries(skel->maps.arr), "cb_map"); + + err = bpf_map__lookup_elem(skel->maps.arr, &key, sizeof(key), &elem, sizeof(elem), 0); + if (ASSERT_OK(err, "lookup")) + ASSERT_EQ(elem.val, 0, "value_cleared"); + + /* The head is disarmed before the callback runs, so it can be reused. */ + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + if (!ASSERT_OK(err, "test_run_again")) + goto out; + ASSERT_EQ(skel->bss->arm_err, 0, "rearm_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +static void test_call_rcu_chain(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + skel->bss->chain = 1; + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts), "test_run")) + goto out; + + if (!wait_for_callbacks(skel, 1)) + goto out; + ASSERT_EQ(skel->bss->chain_err, 0, "chain_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +/* + * A callback that keeps re-arming must stop once the map loses its last user + * reference, or it pins the program for good. Hold an independent fd on .bss + * so the refusal is still readable after the skeleton is gone. + */ +static void test_call_rcu_teardown(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int i, err, fd = 0, bss_fd = -1; + struct bpf_prog_info pinfo = {}; + struct bpf_map_info minfo = {}; + __u32 len, prog_id, zero = 0; + struct call_rcu *skel; + char *buf = NULL; + size_t off, vsz; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + len = sizeof(pinfo); + if (!ASSERT_OK(bpf_prog_get_info_by_fd(bpf_program__fd(skel->progs.arm), &pinfo, &len), + "prog_info")) + goto out; + prog_id = pinfo.id; + + len = sizeof(minfo); + if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &minfo, &len), + "bss_info")) + goto out; + bss_fd = bpf_map_get_fd_by_id(minfo.id); + if (!ASSERT_GE(bss_fd, 0, "bss_fd")) + goto out; + + vsz = bpf_map__value_size(skel->maps.bss); + off = (char *)&skel->bss->chain_err - (char *)skel->bss; + buf = malloc(vsz); + if (!ASSERT_OK_PTR(buf, "buf")) + goto out; + + skel->bss->chain = INT_MAX; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + /* The chain has to be running before the map reference goes away. */ + if (!wait_for_callbacks(skel, 2)) + goto out; + + call_rcu__destroy(skel); + skel = NULL; + + for (i = 0; i < 3000; i++) { + fd = bpf_prog_get_fd_by_id(prog_id); + if (fd < 0) + break; + close(fd); + usleep(10000); + } + if (!ASSERT_EQ(fd, -ENOENT, "prog_freed")) + goto out; + + if (ASSERT_OK(bpf_map_lookup_elem(bss_fd, &zero, buf), "bss_lookup")) + ASSERT_EQ(*(int *)(buf + off), -EPERM, "chain_refused"); +out: + free(buf); + if (bss_fd >= 0) + close(bss_fd); + call_rcu__destroy(skel); +} + +static void test_call_rcu_bad_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.btf_fd = bpf_object__btf_fd(skel->obj); + opts.btf_key_type_id = bpf_map__btf_key_type_id(skel->maps.arr); + opts.btf_value_type_id = bpf_map__btf_value_type_id(skel->maps.arr); + + fd = bpf_map_create(BPF_MAP_TYPE_HASH, "rcu_hash", sizeof(__u32), + bpf_map__value_size(skel->maps.arr), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "hash_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Iterating would hand the program a writable pointer to the head. */ +static void test_call_rcu_iter(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo = {}; + struct bpf_link *link; + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + linfo.map.map_fd = bpf_map__fd(skel->maps.arr); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump, &opts); + if (!ASSERT_ERR_PTR(link, "iter_rejected")) + bpf_link__destroy(link); + else + ASSERT_EQ(libbpf_get_error(link), -EOPNOTSUPP, "iter_errno"); + + call_rcu__destroy(skel); +} + +static void test_call_rcu_inner_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.inner_map_fd = bpf_map__fd(skel->maps.arr); + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "rcu_outer", + sizeof(__u32), sizeof(__u32), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "inner_map_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Two heads in the same map must be independent. */ +static void test_call_rcu_two_heads(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_both), &opts), + "test_run")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + if (!wait_for_callbacks(skel, 2)) + goto out; + ASSERT_EQ(skel->bss->cb_keys, 0x3, "both_keys"); +out: + call_rcu__destroy(skel); +} + +void test_call_rcu(void) +{ + if (test__start_subtest("run")) + test_call_rcu_run(); + if (test__start_subtest("chain")) + test_call_rcu_chain(); + if (test__start_subtest("two_heads")) + test_call_rcu_two_heads(); + if (test__start_subtest("teardown")) + test_call_rcu_teardown(); + if (test__start_subtest("bad_map")) + test_call_rcu_bad_map(); + if (test__start_subtest("iter")) + test_call_rcu_iter(); + if (test__start_subtest("inner_map")) + test_call_rcu_inner_map(); + RUN_TESTS(call_rcu_fail); +} diff --git a/tools/testing/selftests/bpf/progs/call_rcu.c b/tools/testing/selftests/bpf/progs/call_rcu.c new file mode 100644 index 000000000000..94d92cee88a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include + +char _license[] SEC("license") = "GPL"; + +/* rh is not at offset 0, so the callback's value recovery is exercised. */ +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +__u32 cb_key; +__u64 cb_keys; +__u64 cb_val; +__u32 cb_max_entries; +int callbacks; +int arm_err; +int busy_err; +int chain; /* set by userspace: number of times to re-arm from the callback */ +int chain_err; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + cb_key = *(__u32 *)key; + __sync_fetch_and_or(&cb_keys, 1ULL << cb_key); + cb_val = e->val; + cb_max_entries = map->max_entries; + e->val = 0; + + if (chain > 0) { + chain--; + chain_err = bpf_call_rcu(&e->rh, &arr, reclaim); + } + + __sync_fetch_and_add(&callbacks, 1); + return 0; +} + +SEC("syscall") +int arm(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* Keep a grace period from elapsing between the two arms. */ + bpf_rcu_read_lock(); + arm_err = bpf_call_rcu(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu(&e->rh, &arr, reclaim); + bpf_rcu_read_unlock(); + return 0; +} + +SEC("syscall") +int arm_both(void *ctx) +{ + __u32 key0 = 0, key1 = 1; + struct elem *e0, *e1; + + e0 = bpf_map_lookup_elem(&arr, &key0); + e1 = bpf_map_lookup_elem(&arr, &key1); + if (!e0 || !e1) + return 1; + + e0->val = 0xdeadbeef; + e1->val = 0xdeadbeef; + arm_err = bpf_call_rcu(&e0->rh, &arr, reclaim); + arm_err |= bpf_call_rcu(&e1->rh, &arr, reclaim); + return 0; +} + +SEC("iter/bpf_map_elem") +int dump(struct bpf_iter__bpf_map_elem *ctx) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/call_rcu_fail.c b/tools/testing/selftests/bpf/progs/call_rcu_fail.c new file mode 100644 index 000000000000..bf60a27731fb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu_fail.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const void *user_ptr = NULL; + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr2 SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} plain SEC(".maps"); + +__u32 key = 0; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + return 0; +} + +static int sleepable_reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + bpf_copy_from_user(&e->val, sizeof(e->val), user_ptr); + return 0; +} + +SEC("syscall") +__failure __msg("bpf_rcu_head pointer in R1") __msg("doesn't match map pointer in R2") +int mismatch_map(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr2, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("map 'plain' has no valid bpf_rcu_head") +int no_rcu_head(void *ctx) +{ + __u64 *val; + + val = bpf_map_lookup_elem(&plain, &key); + if (!val) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)val, &plain, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("doesn't point to 'struct bpf_rcu_head' that is at 8") +int wrong_offset(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)&e->pad, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("R1 type=fp expected=map_value") +int rcu_head_on_stack(void *ctx) +{ + struct bpf_rcu_head rh; + + bpf_call_rcu(&rh, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_copy_from_user") __msg("in non-sleepable prog") +int sleepable_callback(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr, sleepable_reclaim); + return 0; +} From db1b20ed6b093d98bfc3fb7ced2dce971d4e81ce Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 22 Sep 2026 13:00:52 -0700 Subject: [PATCH 214/660] bpf: Add bpf_call_rcu_tasks_trace() kfunc Sleepable BPF programs hold rcu_read_lock_trace(), not rcu_read_lock(), so a plain RCU grace period does not wait for them. A program whose readers are sleepable needs this flavour to defer reclaim safely. call_rcu_tasks_trace() is call_srcu() on rcu_tasks_trace_srcu_struct and SRCU invokes callbacks with BH disabled, so the callback is still not sleepable. It does run from a kworker rather than softirq or the rcuc/rcuo kthread, so a callback must not assume anything about current. Only the queueing call differs, so the two share struct bpf_rcu_head and all of the verifier plumbing. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922200208.3203834-4-puranjay@kernel.org --- kernel/bpf/helpers.c | 57 +++++++++++++++++++++++++++++++------------ kernel/bpf/verifier.c | 7 ++++-- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 8a01dd4058a0..301b35bd85c8 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4838,21 +4838,10 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu) bpf_prog_put(prog); } -/** - * bpf_call_rcu - Invoke a BPF callback after an RCU grace period - * @rh: struct bpf_rcu_head in a BPF map value - * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values - * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh - * @aux: bpf_prog_aux of the caller, implicitly set by the verifier - * - * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process - * nor bpffs, or -EBADF if the calling program is going away. - */ -__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, - bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +static int __bpf_call_rcu(struct bpf_rcu_head *rh, struct bpf_map *map, void *callback, + struct bpf_prog_aux *aux, bool trace) { struct bpf_rcu_head_kern *rhk = (void *)rh; - struct bpf_map *map = map__const_map; struct bpf_prog *prog; BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head)); @@ -4872,13 +4861,50 @@ __bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, return -EBADF; } - rhk->callback_fn = (bpf_callback_t)(void *)callback; + rhk->callback_fn = (bpf_callback_t)callback; rhk->map = map; rhk->prog = prog; - call_rcu(&rhk->rcu, bpf_rcu_run_callback); + if (trace) + call_rcu_tasks_trace(&rhk->rcu, bpf_rcu_run_callback); + else + call_rcu(&rhk->rcu, bpf_rcu_run_callback); return 0; } +/** + * bpf_call_rcu - Invoke a BPF callback after an RCU grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, false); +} + +/** + * bpf_call_rcu_tasks_trace - Invoke a BPF callback after an RCU tasks trace grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Waits for sleepable BPF programs too. The callback itself is not sleepable either way. + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu_tasks_trace(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, true); +} + static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, struct bpf_dynptr_kern *ptr) { @@ -5175,6 +5201,7 @@ BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu_tasks_trace, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 64db47964ff9..a7c9e2d8965d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -578,7 +578,7 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; - /* bpf_call_rcu callbacks are never sleepable. */ + /* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) return false; @@ -12629,6 +12629,7 @@ enum special_kfunc_type { KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, KF_bpf_call_rcu, + KF_bpf_call_rcu_tasks_trace, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_arena_reserve_pages, @@ -12723,6 +12724,7 @@ BTF_ID(func, __bpf_trap) BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_call_rcu) +BTF_ID(func, bpf_call_rcu_tasks_trace) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) BTF_ID(func, bpf_arena_reserve_pages) @@ -12796,7 +12798,8 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id) static bool is_call_rcu_kfunc(u32 func_id) { - return func_id == special_kfunc_list[KF_bpf_call_rcu]; + return func_id == special_kfunc_list[KF_bpf_call_rcu] || + func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace]; } static bool is_task_work_add_kfunc(u32 func_id) From b22f65a009ad2fff81eca699fb591b89b4980a37 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 22 Sep 2026 13:00:53 -0700 Subject: [PATCH 215/660] selftests/bpf: Add a test for bpf_call_rcu_tasks_trace() Run the same functional test against the tasks trace flavour. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922200208.3203834-5-puranjay@kernel.org --- .../testing/selftests/bpf/prog_tests/call_rcu.c | 12 ++++++++---- tools/testing/selftests/bpf/progs/call_rcu.c | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/call_rcu.c b/tools/testing/selftests/bpf/prog_tests/call_rcu.c index bfa6512b9052..e4d271d43d87 100644 --- a/tools/testing/selftests/bpf/prog_tests/call_rcu.c +++ b/tools/testing/selftests/bpf/prog_tests/call_rcu.c @@ -28,9 +28,10 @@ static bool wait_for_callbacks(struct call_rcu *skel, int expected) return ASSERT_GE(got, expected, "callbacks"); } -static void test_call_rcu_run(void) +static void test_call_rcu_run(bool trace) { LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_program *prog; struct call_rcu *skel; struct elem elem; __u32 key = 1; @@ -40,7 +41,8 @@ static void test_call_rcu_run(void) if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) return; - err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + prog = trace ? skel->progs.arm_trace : skel->progs.arm; + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) goto out; @@ -59,7 +61,7 @@ static void test_call_rcu_run(void) ASSERT_EQ(elem.val, 0, "value_cleared"); /* The head is disarmed before the callback runs, so it can be reused. */ - err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); if (!ASSERT_OK(err, "test_run_again")) goto out; ASSERT_EQ(skel->bss->arm_err, 0, "rearm_err"); @@ -254,7 +256,9 @@ out: void test_call_rcu(void) { if (test__start_subtest("run")) - test_call_rcu_run(); + test_call_rcu_run(false); + if (test__start_subtest("run_tasks_trace")) + test_call_rcu_run(true); if (test__start_subtest("chain")) test_call_rcu_chain(); if (test__start_subtest("two_heads")) diff --git a/tools/testing/selftests/bpf/progs/call_rcu.c b/tools/testing/selftests/bpf/progs/call_rcu.c index 94d92cee88a7..505008647fd6 100644 --- a/tools/testing/selftests/bpf/progs/call_rcu.c +++ b/tools/testing/selftests/bpf/progs/call_rcu.c @@ -86,6 +86,23 @@ int arm_both(void *ctx) return 0; } +SEC("syscall") +int arm_trace(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* No lock needed: the enclosing rcu_read_lock_trace() already blocks the grace period. */ + arm_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + return 0; +} + SEC("iter/bpf_map_elem") int dump(struct bpf_iter__bpf_map_elem *ctx) { From a2710072ddeca1b2e0962f946f36f0b2804c9aa4 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Fri, 11 Sep 2026 17:33:38 -0400 Subject: [PATCH 216/660] bpf, mips: Factor out div/mod emission helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor MIPS32 and MIPS64 division and modulo emission out of emit_alu_r() and emit_alu_r64(). This prepares the JITs to select signed or unsigned opcodes without duplicating the R6 and pre-R6 handling. No functional change intended. Suggested-by: Philippe Mathieu-Daudé Signed-off-by: Nicholas Dudar Signed-off-by: Alexei Starovoitov Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260911213340.3767930-2-main.kalliope@gmail.com --- arch/mips/net/bpf_jit_comp.c | 36 ++++++++++++++++++++++------------ arch/mips/net/bpf_jit_comp64.c | 36 ++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index 6ee4abe6a1f7..320180330fb3 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -338,6 +338,28 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (32-bit) */ +static void emit_div(struct jit_context *ctx, u8 dst, u8 src) +{ + if (cpu_has_mips32r6) { + emit(ctx, divu_r6, dst, dst, src); + } else { + emit(ctx, divu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (32-bit) */ +static void emit_mod(struct jit_context *ctx, u8 dst, u8 src) +{ + if (cpu_has_mips32r6) { + emit(ctx, modu, dst, dst, src); + } else { + emit(ctx, divu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (32-bit) */ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) { @@ -385,21 +407,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips32r6) { - emit(ctx, divu_r6, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mflo, dst); - } + emit_div(ctx, dst, src); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips32r6) { - emit(ctx, modu, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod(ctx, dst, src); break; } clobber_reg(ctx, dst); diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index fa7e9aa37f49..2520e1db7ab7 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -197,6 +197,28 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (64-bit) */ +static void emit_div64(struct jit_context *ctx, u8 dst, u8 src) +{ + if (cpu_has_mips64r6) { + emit(ctx, ddivu_r6, dst, dst, src); + } else { + emit(ctx, ddivu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (64-bit) */ +static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src) +{ + if (cpu_has_mips64r6) { + emit(ctx, dmodu, dst, dst, src); + } else { + emit(ctx, ddivu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (64-bit) */ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) { @@ -235,21 +257,11 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips64r6) { - emit(ctx, ddivu_r6, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mflo, dst); - } + emit_div64(ctx, dst, src); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips64r6) { - emit(ctx, dmodu, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod64(ctx, dst, src); break; default: /* Width-generic operations */ From 668508db5ecb386cae79a2d34a057d48d7469a65 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Fri, 11 Sep 2026 17:33:39 -0400 Subject: [PATCH 217/660] MIPS: uasm: Add signed div/mod and sign-extension emitters Add uasm emitters for the signed DIV, MOD, DDIV, and DMOD instructions. Provide both pre-R6 and R6 division forms so users can select the instruction appropriate for the target ISA. Add SEB and SEH emitters for R2-or-newer CPUs. These will be used by the follow-on BPF_MOVSX series. Signed-off-by: Nicholas Dudar Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/CAM1=_QS79dDBfaaNQXnWd61AqHd1M4o9aDMMiftnoJveNr=FZg@mail.gmail.com/ Link: https://lore.kernel.org/bpf/CAM1=_QTN3JJ6jtckf32F8wxBzLDbHSrzUttyMBWNqZKmRe=2rg@mail.gmail.com/ Link: https://patch.msgid.link/20260911213340.3767930-3-main.kalliope@gmail.com --- arch/mips/include/asm/uasm.h | 8 ++++++++ arch/mips/mm/uasm-mips.c | 12 ++++++++++++ arch/mips/mm/uasm.c | 24 +++++++++++++++++------- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index b43bfd445252..adea35df0f8d 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -85,15 +85,20 @@ Ip_u1u2(_ctc1); Ip_u2u1(_ctcmsa); Ip_u2u1s3(_daddiu); Ip_u3u1u2(_daddu); +Ip_u1u2(_ddiv); +Ip_u3u1u2(_ddiv_r6); Ip_u1u2(_ddivu); Ip_u3u1u2(_ddivu_r6); Ip_u1(_di); Ip_u2u1msbu3(_dins); Ip_u2u1msbu3(_dinsm); Ip_u2u1msbu3(_dinsu); +Ip_u1u2(_div); +Ip_u3u1u2(_div_r6); Ip_u1u2(_divu); Ip_u3u1u2(_divu_r6); Ip_u1u2u3(_dmfc0); +Ip_u3u1u2(_dmod); Ip_u3u1u2(_dmodu); Ip_u1u2u3(_dmtc0); Ip_u1u2(_dmultu); @@ -135,6 +140,7 @@ Ip_u1u2u3(_mfc0); Ip_u1u2u3(_mfhc0); Ip_u1(_mfhi); Ip_u1(_mflo); +Ip_u3u1u2(_mod); Ip_u3u1u2(_modu); Ip_u3u1u2(_movn); Ip_u3u1u2(_movz); @@ -156,6 +162,8 @@ Ip_u2s3u1(_sb); Ip_u2s3u1(_sc); Ip_u2s3u1(_scd); Ip_u2s3u1(_sd); +Ip_u2u1(_seb); +Ip_u2u1(_seh); Ip_u3u1u2(_seleqz); Ip_u3u1u2(_selnez); Ip_u2s3u1(_sh); diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index e15c6700cd08..2d1668b58ddd 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -75,6 +75,9 @@ static const struct insn insn_table[insn_invalid] = { [insn_ctcmsa] = {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE}, [insn_daddiu] = {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, [insn_daddu] = {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD}, + [insn_ddiv] = {M(spec_op, 0, 0, 0, 0, ddiv_op), RS | RT}, + [insn_ddiv_r6] = {M(spec_op, 0, 0, 0, ddiv_ddiv6_op, ddiv_op), + RS | RT | RD}, [insn_ddivu] = {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT}, [insn_ddivu_r6] = {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op), RS | RT | RD}, @@ -82,10 +85,15 @@ static const struct insn insn_table[insn_invalid] = { [insn_dins] = {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE}, [insn_dinsm] = {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE}, [insn_dinsu] = {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE}, + [insn_div] = {M(spec_op, 0, 0, 0, 0, div_op), RS | RT}, + [insn_div_r6] = {M(spec_op, 0, 0, 0, div_div6_op, div_op), + RS | RT | RD}, [insn_divu] = {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT}, [insn_divu_r6] = {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op), RS | RT | RD}, [insn_dmfc0] = {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmod] = {M(spec_op, 0, 0, 0, ddiv_dmod_op, ddiv_op), + RS | RT | RD}, [insn_dmodu] = {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op), RS | RT | RD}, [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, @@ -140,6 +148,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_mfhc0] = {M(cop0_op, mfhc0_op, 0, 0, 0, 0), RT | RD | SET}, [insn_mfhi] = {M(spec_op, 0, 0, 0, 0, mfhi_op), RD}, [insn_mflo] = {M(spec_op, 0, 0, 0, 0, mflo_op), RD}, + [insn_mod] = {M(spec_op, 0, 0, 0, div_mod_op, div_op), + RS | RT | RD}, [insn_modu] = {M(spec_op, 0, 0, 0, divu_modu_op, divu_op), RS | RT | RD}, [insn_movn] = {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD}, @@ -177,6 +187,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_scd] = {M6(spec3_op, 0, 0, 0, scd6_op), RS | RT | SIMM9}, #endif [insn_sd] = {M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, + [insn_seb] = {M(spec3_op, 0, 0, 0, seb_op, bshfl_op), RT | RD}, + [insn_seh] = {M(spec3_op, 0, 0, 0, seh_op, bshfl_op), RT | RD}, [insn_seleqz] = {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD}, [insn_selnez] = {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD}, [insn_sh] = {M(sh_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 125140979d62..139afb0e14cd 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c @@ -49,19 +49,21 @@ enum opcode { insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1, insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez, insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1, - insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu, - insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu, - insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu, + insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddiv, + insn_ddiv_r6, insn_ddivu, insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, + insn_dinsu, insn_div, insn_div_r6, insn_divu, insn_divu_r6, insn_dmfc0, + insn_dmod, insn_dmodu, insn_dmtc0, insn_dmultu, insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll, insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl, insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins, insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld, insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, - insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, - insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, - insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, - insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_mflo, insn_mod, insn_modu, insn_movn, insn_movz, insn_mtc0, + insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, + insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, + insn_sb, insn_sc, insn_scd, insn_seb, insn_seh, + insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, @@ -287,9 +289,12 @@ I_u1u2(_cfc1) I_u2u1(_cfcmsa) I_u1u2(_ctc1) I_u2u1(_ctcmsa) +I_u1u2(_ddiv) +I_u3u1u2(_ddiv_r6) I_u1u2(_ddivu) I_u3u1u2(_ddivu_r6) I_u1u2u3(_dmfc0) +I_u3u1u2(_dmod) I_u3u1u2(_dmodu) I_u1u2u3(_dmtc0) I_u1u2(_dmultu) @@ -297,6 +302,8 @@ I_u3u1u2(_dmulu) I_u2u1s3(_daddiu) I_u3u1u2(_daddu) I_u1(_di); +I_u1u2(_div) +I_u3u1u2(_div_r6) I_u1u2(_divu) I_u3u1u2(_divu_r6) I_u2u1(_dsbh); @@ -332,6 +339,7 @@ I_u2s3u1(_lw) I_u2s3u1(_lwu) I_u1u2u3(_mfc0) I_u1u2u3(_mfhc0) +I_u3u1u2(_mod) I_u3u1u2(_modu) I_u3u1u2(_movn) I_u3u1u2(_movz) @@ -353,6 +361,8 @@ I_u2s3u1(_sb) I_u2s3u1(_sc) I_u2s3u1(_scd) I_u2s3u1(_sd) +I_u2u1(_seb) +I_u2u1(_seh) I_u3u1u2(_seleqz) I_u3u1u2(_selnez) I_u2s3u1(_sh) From 47c26a8570d10d0de46aec3f311d38db653d3572 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Fri, 11 Sep 2026 17:33:40 -0400 Subject: [PATCH 218/660] bpf, mips: Add support for BPF_SDIV and BPF_SMOD The MIPS JITs handle BPF_DIV and BPF_MOD without inspecting insn->off, which distinguishes BPF_SDIV and BPF_SMOD. Signed operations therefore use unsigned instructions, or unsigned helpers for ALU64 on 32-bit MIPS, and produce unsigned results for negative operands. Pass insn->off unchanged through immediate validation and register emission, and interpret it only in div/mod-specific paths. Use signed DIV/DDIV for pre-R6 and DIV/MOD/DDIV/DMOD for R6. On 32-bit MIPS, use div64_s64() and a signed remainder helper for ALU64. Keep the signed quotient signed while computing that remainder, and keep signed immediates out of the unsigned power-of-two shift/mask rewrite. check_alu_op() rejects immediate zero divisors. bpf_do_misc_fixups() rewrites signed immediate -1 and guards register divisors that are zero or, for signed operations, -1 before JIT compilation. The ALU32 paths continue to zero-extend their results. Found while auditing BPF JIT handling of signed div/mod. Signed-off-by: Nicholas Dudar Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/CAM1=_QS79dDBfaaNQXnWd61AqHd1M4o9aDMMiftnoJveNr=FZg@mail.gmail.com/ Link: https://lore.kernel.org/bpf/CAM1=_QQ7XiJEozaetLt_+kVn91YAXWpJK_pyfT9=paXseW=wwA@mail.gmail.com/ Link: https://patch.msgid.link/20260911213340.3767930-4-main.kalliope@gmail.com --- arch/mips/net/bpf_jit_comp.c | 35 ++++++++++++++++------- arch/mips/net/bpf_jit_comp.h | 4 +-- arch/mips/net/bpf_jit_comp32.c | 29 ++++++++++++------- arch/mips/net/bpf_jit_comp64.c | 51 +++++++++++++++++++++------------- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index 320180330fb3..f14de6d73122 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -208,7 +208,7 @@ void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) } /* Validate ALU immediate range */ -bool valid_alu_i(u8 op, s32 imm) +bool valid_alu_i(u8 op, s32 imm, s16 off) { switch (BPF_OP(op)) { case BPF_NEG: @@ -237,6 +237,9 @@ bool valid_alu_i(u8 op, s32 imm) return imm == 0 || (imm > 0 && is_power_of_2(imm)); case BPF_DIV: case BPF_MOD: + /* Do not use unsigned shift/mask rewrites for signed div/mod. */ + if (off == 1) + return false; /* imm must be an 17-bit power of two */ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); } @@ -339,29 +342,41 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) } /* ALU division operation (32-bit) */ -static void emit_div(struct jit_context *ctx, u8 dst, u8 src) +static void emit_div(struct jit_context *ctx, u8 dst, u8 src, s16 off) { if (cpu_has_mips32r6) { - emit(ctx, divu_r6, dst, dst, src); + if (off == 1) + emit(ctx, div_r6, dst, dst, src); + else + emit(ctx, divu_r6, dst, dst, src); } else { - emit(ctx, divu, dst, src); + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); emit(ctx, mflo, dst); } } /* ALU modulo operation (32-bit) */ -static void emit_mod(struct jit_context *ctx, u8 dst, u8 src) +static void emit_mod(struct jit_context *ctx, u8 dst, u8 src, s16 off) { if (cpu_has_mips32r6) { - emit(ctx, modu, dst, dst, src); + if (off == 1) + emit(ctx, mod, dst, dst, src); + else + emit(ctx, modu, dst, dst, src); } else { - emit(ctx, divu, dst, src); + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); emit(ctx, mfhi, dst); } } /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off) { switch (BPF_OP(op)) { /* dst = dst & src */ @@ -407,11 +422,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - emit_div(ctx, dst, src); + emit_div(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - emit_mod(ctx, dst, src); + emit_mod(ctx, dst, src, off); break; } clobber_reg(ctx, dst); diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h index a37fe20818eb..0ddedb733dd1 100644 --- a/arch/mips/net/bpf_jit_comp.h +++ b/arch/mips/net/bpf_jit_comp.h @@ -163,7 +163,7 @@ void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); /* Validate ALU/ALU64 immediate range */ -bool valid_alu_i(u8 op, s32 imm); +bool valid_alu_i(u8 op, s32 imm, s16 off); /* Rewrite ALU/ALU64 immediate operation */ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); @@ -172,7 +172,7 @@ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off); /* Atomic read-modify-write (32-bit) */ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 40a878b672f5..6619182b5ef3 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -512,7 +512,7 @@ static void emit_mul_r64(struct jit_context *ctx, clobber_reg64(ctx, dst); } -/* Helper function for 64-bit modulo */ +/* Helper function for unsigned 64-bit modulo */ static u64 jit_mod64(u64 a, u64 b) { u64 rem; @@ -521,13 +521,22 @@ static u64 jit_mod64(u64 a, u64 b) return rem; } +/* Helper function for signed 64-bit modulo */ +static s64 jit_smod64(s64 a, s64 b) +{ + s64 quot = div64_s64(a, b); + + return a - quot * b; +} + /* ALU div/mod register (64-bit) */ -static void emit_divmod_r64(struct jit_context *ctx, - const u8 dst[], const u8 src[], u8 op) +static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[], u8 op, s16 off) { const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ + bool is_signed = off == 1; int exclude, k; u32 addr = 0; @@ -546,11 +555,11 @@ static void emit_divmod_r64(struct jit_context *ctx, switch (BPF_OP(op)) { /* dst = dst / src */ case BPF_DIV: - addr = (u32)&div64_u64; + addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64; break; /* dst = dst % src */ case BPF_MOD: - addr = (u32)&jit_mod64; + addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64; break; } emit_mov_i(ctx, MIPS_R_T9, addr); @@ -1516,9 +1525,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T6, imm); - emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); + emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, lo(dst), val, alu); } @@ -1546,7 +1555,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_X: - emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); + emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -1599,7 +1608,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) * and then do the operation on this register. */ emit_mov_se_i64(ctx, tmp, imm); - emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); + emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off); break; /* dst = dst & src (64-bit) */ /* dst = dst | src (64-bit) */ @@ -1629,7 +1638,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* dst = dst % src (64-bit) */ case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_divmod_r64(ctx, dst, src, BPF_OP(code)); + emit_divmod_r64(ctx, dst, src, BPF_OP(code), off); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 2520e1db7ab7..db027c2d9221 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -198,29 +198,42 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) } /* ALU division operation (64-bit) */ -static void emit_div64(struct jit_context *ctx, u8 dst, u8 src) +static void emit_div64(struct jit_context *ctx, u8 dst, u8 src, s16 off) { if (cpu_has_mips64r6) { - emit(ctx, ddivu_r6, dst, dst, src); + if (off == 1) + emit(ctx, ddiv_r6, dst, dst, src); + else + emit(ctx, ddivu_r6, dst, dst, src); } else { - emit(ctx, ddivu, dst, src); + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); emit(ctx, mflo, dst); } } /* ALU modulo operation (64-bit) */ -static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src) +static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src, s16 off) { if (cpu_has_mips64r6) { - emit(ctx, dmodu, dst, dst, src); + if (off == 1) + emit(ctx, dmod, dst, dst, src); + else + emit(ctx, dmodu, dst, dst, src); } else { - emit(ctx, ddivu, dst, src); + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); emit(ctx, mfhi, dst); } } /* ALU register operation (64-bit) */ -static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) +static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op, + s16 off) { switch (BPF_OP(op)) { /* dst = dst << src */ @@ -257,15 +270,15 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - emit_div64(ctx, dst, src); + emit_div64(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - emit_mod64(ctx, dst, src); + emit_mod64(ctx, dst, src, off); break; default: /* Width-generic operations */ - emit_alu_r(ctx, dst, src, op); + emit_alu_r(ctx, dst, src, op, off); } clobber_reg(ctx, dst); } @@ -686,9 +699,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_LSH | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, dst, val, alu); } @@ -708,10 +721,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_sext(ctx, dst, dst); emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_sext(ctx, dst, dst); emit_alu_i(ctx, dst, val, alu); @@ -726,7 +739,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_LSH | BPF_X: - emit_alu_r(ctx, dst, src, BPF_OP(code)); + emit_alu_r(ctx, dst, src, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = dst >> src */ @@ -745,7 +758,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MOD | BPF_X: emit_sext(ctx, dst, dst); emit_sext(ctx, MIPS_R_T4, src); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -782,9 +795,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i64(ctx, dst, val, alu); } @@ -811,7 +824,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_alu_r64(ctx, dst, src, BPF_OP(code)); + emit_alu_r64(ctx, dst, src, BPF_OP(code), off); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ From 86187bd3cd5a15b3f5b8023ef7123abc919c9bfe Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Sat, 12 Sep 2026 00:12:19 -0400 Subject: [PATCH 219/660] bpf, mips: Factor register moves into helpers Factor ALU32 and ALU64 register moves into backend-local helpers, keeping register-pair handling and zero extension with the move. This prepares for decoding the MOVSX width in those helpers. Keep the verifier-inserted zero-extension marker on its existing path. No functional change intended. Signed-off-by: Nicholas Dudar Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260912041220.444715-2-main.kalliope@gmail.com --- arch/mips/net/bpf_jit_comp32.c | 22 ++++++++++++++++++---- arch/mips/net/bpf_jit_comp64.c | 18 +++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 6619182b5ef3..6d603d9eeb1a 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -190,6 +190,22 @@ static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) } } +/* Register move operation (32-bit) */ +static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], + const u8 src[]) +{ + emit_mov_r(ctx, lo(dst), lo(src)); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit) */ +static void emit_mov_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[]) +{ + emit_mov_r(ctx, lo(dst), lo(src)); + emit_mov_r(ctx, hi(dst), hi(src)); +} + /* Load delay slot, if ISA mandates it */ static void emit_load_delay(struct jit_context *ctx) { @@ -1494,8 +1510,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_mov_i(ctx, hi(dst), 0); } else { - emit_mov_r(ctx, lo(dst), lo(src)); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src); } break; /* dst = -dst */ @@ -1564,8 +1579,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, lo(dst), lo(src)); - emit_mov_r(ctx, hi(dst), hi(src)); + emit_mov_r64(ctx, dst, src); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index db027c2d9221..92df2eba1390 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -120,6 +120,19 @@ static void emit_zext_ver(struct jit_context *ctx, u8 dst) emit_zext(ctx, dst); } +/* Register move operation (32-bit) */ +static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src) +{ + emit_mov_r(ctx, dst, src); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit) */ +static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src) +{ + emit_mov_r(ctx, dst, src); +} + /* dst = imm (64-bit) */ static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) { @@ -681,8 +694,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_zext(ctx, dst); } else { - emit_mov_r(ctx, dst, src); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src); } break; /* dst = -dst */ @@ -767,7 +779,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, dst, src); + emit_mov_r64(ctx, dst, src); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: From 07ebb8c386d384d69e6f2c17304f9cb381d4e55a Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Sat, 12 Sep 2026 00:12:20 -0400 Subject: [PATCH 220/660] bpf, mips: Add support for BPF_MOVSX in the JITs Both MIPS JITs ignore insn->off when lowering register MOV instructions, so BPF_MOVSX is emitted as an ordinary move. Since build_insn() accepts it, affected programs are silently miscompiled instead of falling back to the interpreter. Decode the MOVSX width from insn->off in the register-move helpers. On MIPS32, propagate the sign into the high word for ALU64 while preserving the existing ALU32 zero-extension handling. Use seb/seh on MIPS32 R2-or-newer CPUs and shift pairs otherwise. On MIPS64, seb/seh require a sign-extended 32-bit source. Normalizing an arbitrary BPF register first would cost another instruction, so retain the doubleword shift pairs. Signed-off-by: Nicholas Dudar Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260912041220.444715-3-main.kalliope@gmail.com --- arch/mips/net/bpf_jit_comp32.c | 67 +++++++++++++++++++++++++++++----- arch/mips/net/bpf_jit_comp64.c | 55 ++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 6d603d9eeb1a..8f1ab851863f 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -190,20 +190,67 @@ static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) } } -/* Register move operation (32-bit) */ -static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], - const u8 src[]) +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) { - emit_mov_r(ctx, lo(dst), lo(src)); + int shift; + + if (cpu_has_mips32r2 || cpu_has_mips32r6) { + if (off == 8) + emit(ctx, seb, dst, src); + else + emit(ctx, seh, dst, src); + } else { + shift = 32 - off; + emit(ctx, sll, dst, src, shift); + emit(ctx, sra, dst, dst, shift); + } +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], + const u8 src[], s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, lo(dst), lo(src)); + break; + } + clobber_reg(ctx, lo(dst)); emit_zext_ver(ctx, dst); } -/* Register move operation (64-bit) */ +/* Register move operation (64-bit), optionally with sign extension */ static void emit_mov_r64(struct jit_context *ctx, const u8 dst[], - const u8 src[]) + const u8 src[], s16 off) { - emit_mov_r(ctx, lo(dst), lo(src)); - emit_mov_r(ctx, hi(dst), hi(src)); + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + case 32: + emit(ctx, move, lo(dst), lo(src)); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, lo(dst), lo(src)); + emit_mov_r(ctx, hi(dst), hi(src)); + break; + } + clobber_reg64(ctx, dst); } /* Load delay slot, if ISA mandates it */ @@ -1510,7 +1557,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_mov_i(ctx, hi(dst), 0); } else { - emit_mov_r32(ctx, dst, src); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -1579,7 +1626,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r64(ctx, dst, src); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 92df2eba1390..4507841a9224 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -120,17 +120,56 @@ static void emit_zext_ver(struct jit_context *ctx, u8 dst) emit_zext(ctx, dst); } -/* Register move operation (32-bit) */ -static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src) +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) { - emit_mov_r(ctx, dst, src); + /* seb/seh would need a sign-extended 32-bit source first. */ + int shift = 32 - off; + + /* dsll32 and dsra32 add 32 to the shift argument. */ + emit(ctx, dsll32, dst, src, shift); + emit(ctx, dsra32, dst, dst, shift); +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); emit_zext_ver(ctx, dst); } -/* Register move operation (64-bit) */ -static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src) +/* Register move operation (64-bit), optionally with sign extension */ +static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) { - emit_mov_r(ctx, dst, src); + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + case 32: + emit_sext(ctx, dst, src); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); } /* dst = imm (64-bit) */ @@ -694,7 +733,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_zext(ctx, dst); } else { - emit_mov_r32(ctx, dst, src); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -779,7 +818,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r64(ctx, dst, src); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: From ddaa827dcea4544ccfffa1dd85ab0ea35cd06e6d Mon Sep 17 00:00:00 2001 From: Madhav Khosla Date: Sun, 20 Sep 2026 14:25:49 +0530 Subject: [PATCH 221/660] selftests/bpf: Fix csum_partial() dropping trailing byte on odd length csum_partial() computes num_u16 = len >> 1 and only sums that many 16-bit words, so the last byte of an odd-length buffer never gets added to the checksum. RFC 1071 says it should be padded with a zero byte and summed as one more word, not dropped. This backs build_ip_csum(), build_udp_v4_csum() and build_udp_v6_csum(), used by flow_dissector_classification.c and xdp_metadata.c to hand-build packets. No current caller builds an odd-length payload, so nothing fails today, but a future one would get a silently wrong checksum. Also bump flow_dissector_classification's TEST_PACKET_LEN from 100 to 99 so this actually gets exercised instead of staying latent. f4504af68575 wrote the len >> 1 division, but it only ever passed sizeof(iphdr), always even, so it couldn't hit the bug. Tagging bcc00987bc56 instead, since it added the first caller, build_udp_v4_csum()/build_udp_v6_csum(), that can pass an odd length. Verified with test_progs under vmtest.sh: without the fix, an odd TEST_PACKET_LEN makes the kernel drop the packet over a bad checksum and flow_dissector_classification fails; with the fix, both flow_dissector_classification and xdp_metadata pass. Signed-off-by: Madhav Khosla Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260920085549.867099-1-madhav.khoslaa@gmail.com --- tools/testing/selftests/bpf/network_helpers.h | 15 +++++++++++++-- .../prog_tests/flow_dissector_classification.c | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 75133119c04a..878c9fc5c37c 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -129,12 +129,23 @@ static __u16 csum_fold(__u32 csum) static __wsum csum_partial(const void *buf, int len, __wsum sum) { - __u16 *p = (__u16 *)buf; + const __u8 *p = buf; int num_u16 = len >> 1; int i; for (i = 0; i < num_u16; i++) - sum += p[i]; + sum += ((const __u16 *)p)[i]; + + /* + * RFC 1071: an odd-length buffer's trailing byte is paired with + * a zero pad byte to form the final 16-bit word. + */ + if (len & 1) { + __u16 tail = 0; + + __builtin_memcpy(&tail, p + len - 1, 1); + sum += tail; + } return sum; } diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c index 80b153d3ddec..421dfa6c4ea3 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c @@ -27,7 +27,7 @@ #define TEST_NAME_MAX_LEN (32 + SUBTEST_NAME_MAX_LEN) #define MAX_SOURCE_PORTS 3 #define TEST_PACKETS_COUNT 10 -#define TEST_PACKET_LEN 100 +#define TEST_PACKET_LEN 99 #define TEST_PACKET_PATTERN 'a' #define TEST_IPV4 "192.168.0.1/32" #define TEST_IPV6 "100::a/128" From c7cd8be3d72f9b03454424fff31542b8a0122437 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 22 Sep 2026 18:17:33 +0800 Subject: [PATCH 222/660] bpf: arena: Allocate the fault-in page outside the lock arena_vm_fault() allocated the page while holding arena->spinlock, so it could only use the non-blocking allocator, which never reclaims. Once the memcg is at memory.max that allocation just fails, the fault turns into VM_FAULT_SIGSEGV, and the process gets a SIGSEGV on a perfectly valid arena address. Hitting memory.max is routine (e.g. page cache from reading a big file), so this kills innocent processes over memory that reclaim could have freed. Rework the fault handler: - Preallocate the page before taking the lock, like do_anonymous_page() does, so it can sleep and reclaim, instead of turning a routine memory.max into a fake segfault. The allocation uses __GFP_RETRY_MAYFAIL so it never invokes the OOM killer: the page is charged to the map's memcg, which need not be the faulting task's, so an OOM there could kill unrelated tasks in the map's cgroup while a foreign faulter could never be its victim. map->numa_node is always NUMA_NO_NODE for an arena (BPF_F_NUMA_NODE is rejected), so the page comes from the local node, as before. - On allocation failure fall through to the locked recheck rather than failing right away: a page a concurrent allocator installed meanwhile is used, otherwise the non-blocking fallback fails and we return VM_FAULT_SIGBUS. Not VM_FAULT_OOM: nothing ran the OOM killer, and the fault path would just retry it forever. - A lockless probe skips that preallocation when a page is already mapped (e.g. allocated by the bpf program), so the common case wastes no allocation. The rare race where such a page is freed before we take the lock falls back to the non-blocking allocator under the lock. - Return VM_FAULT_SIGBUS for the other non-recoverable errors (allocation, range-tree and page-table failures) instead of VM_FAULT_SIGSEGV; the lock failure already did. Only BPF_F_SEGV_ON_FAULT, and a scratch-page hole under that flag, is a real user addressing error and keeps VM_FAULT_SIGSEGV. - Tidy up the error labels. Signed-off-by: Jiayuan Chen Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260922101831.192102-2-jiayuan.chen@linux.dev --- kernel/bpf/arena.c | 83 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..c6369ea5e208 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -481,7 +481,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; - struct page *page; + struct page *page, *new_page = NULL; + vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; int ret; @@ -489,59 +490,101 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + page = vmalloc_to_page((void *)kaddr); + if (!page && !(arena->map.map_flags & BPF_F_SEGV_ON_FAULT)) { + /* + * Preallocate outside the lock so the allocation can reclaim; + * __GFP_RETRY_MAYFAIL keeps the OOM killer out of it. + */ + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + new_page = alloc_pages_node(map->numa_node, + GFP_KERNEL | __GFP_ZERO | + __GFP_ACCOUNT | __GFP_NOWARN | + __GFP_RETRY_MAYFAIL, 0); + bpf_map_memcg_exit(old_memcg, new_memcg); + } + + if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ + if (new_page) + free_pages_nolock(new_page, 0); return VM_FAULT_SIGBUS; + } page = vmalloc_to_page((void *)kaddr); if (page) { - if (page == arena->scratch_page) + if (page == arena->scratch_page) { /* BPF triggered scratch here; don't lazy-alloc over it */ - goto out_sigsegv; + fault_ret = (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) + ? VM_FAULT_SIGSEGV : VM_FAULT_SIGBUS; + goto out_err_locked; + } /* already have a page vmap-ed */ goto out; } + if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) { + /* + * User space requested to segfault when page is not allocated + * by bpf prog + */ + fault_ret = VM_FAULT_SIGSEGV; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) - /* User space requested to segfault when page is not allocated by bpf prog */ - goto out_sigsegv_memcg; + if (!new_page) { + /* + * The probed page was freed meanwhile or preallocation failed; + * try the non-blocking allocator, we cannot sleep here. + */ + ret = bpf_map_alloc_pages(map, map->numa_node, 1, &new_page); + if (ret) { + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; + } + } ret = range_tree_clear(&arena->rt, vmf->pgoff, 1); - if (ret) - goto out_sigsegv_memcg; - - struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; - /* Account into memcg of the process that created bpf_arena */ - ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { - range_tree_set(&arena->rt, vmf->pgoff, 1); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } + struct apply_range_data data = { + .arena = arena, .pages = &new_page, .i = 0 + }; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { range_tree_set(&arena->rt, vmf->pgoff, 1); - free_pages_nolock(page, 0); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } flush_vmap_cache(kaddr, PAGE_SIZE); bpf_map_memcg_exit(old_memcg, new_memcg); + /* new_page was consumed */ + page = new_page; + new_page = NULL; out: page_ref_add(page, 1); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + if (new_page) + free_pages_nolock(new_page, 0); vmf->page = page; return 0; -out_sigsegv_memcg: + +out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); -out_sigsegv: +out_err_locked: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - return VM_FAULT_SIGSEGV; + if (new_page) + free_pages_nolock(new_page, 0); + return fault_ret; } static const struct vm_operations_struct arena_vm_ops = { From a781db31b484f978e017f09234a99cf967949cd7 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 22 Sep 2026 18:17:34 +0800 Subject: [PATCH 223/660] selftests/bpf: Add read_cgroup_file() to cgroup_helpers cgroup_helpers has write_cgroup_file()/write_cgroup_file_parent() but no read counterpart. Add read_cgroup_file() and read_cgroup_file_parent() so a forked child can read a cgroup file (e.g. memory.current) from the work dir owned by the parent that set the environment up, without hand-building the /mnt/... path. Signed-off-by: Jiayuan Chen Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260922101831.192102-3-jiayuan.chen@linux.dev --- tools/testing/selftests/bpf/cgroup_helpers.c | 67 ++++++++++++++++++++ tools/testing/selftests/bpf/cgroup_helpers.h | 4 ++ 2 files changed, 71 insertions(+) diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 45cd0b479fe3..4183ff6150c2 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -188,6 +188,73 @@ int write_cgroup_file_parent(const char *relative_path, const char *file, return __write_cgroup_file(cgroup_path, file, buf); } +static int __read_cgroup_file(const char *cgroup_path, const char *file, + char *buf, size_t len) +{ + char file_path[PATH_MAX + 1]; + ssize_t got; + int fd; + + snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file); + fd = open(file_path, O_RDONLY); + if (fd < 0) { + log_err("Opening %s", file_path); + return 1; + } + + got = read(fd, buf, len - 1); + if (got < 0) { + log_err("Reading %s", file_path); + close(fd); + return 1; + } + buf[got] = '\0'; + close(fd); + return 0; +} + +/** + * read_cgroup_file() - Read from a cgroup file + * @relative_path: The cgroup path, relative to the workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory. + * + * If successful, 0 is returned. + */ +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + +/** + * read_cgroup_file_parent() - Read from a cgroup file in the parent process + * workdir + * @relative_path: The cgroup path, relative to the parent process workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory under the parent process + * workdir. + * + * If successful, 0 is returned. + */ +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_parent_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + /** * setup_cgroup_environment() - Setup the cgroup environment * diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index 3857304be874..d42d2e13044e 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -15,6 +15,10 @@ int write_cgroup_file(const char *relative_path, const char *file, const char *buf); int write_cgroup_file_parent(const char *relative_path, const char *file, const char *buf); +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len); +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len); int cgroup_setup_and_join(const char *relative_path); int get_root_cgroup(void); int create_and_get_cgroup(const char *relative_path); From 1fe5a5cd01e4a897f619640f3e7707015502d81e Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 22 Sep 2026 18:17:35 +0800 Subject: [PATCH 224/660] selftests/bpf: Add a test for arena fault-in under memory.max A child joins a memcg, fills it with 128M of clean page cache by reading a sparse temp file (the way the cgroup selftests do), caps memory.max 8M above its usage and then faults 64M of arena in, which only fits by reclaiming that cache. With the fix the arena fault-in reclaims, every fault succeeds and the child exits 0. Without it the allocation cannot reclaim, fails once the headroom is used up, and the child dies with SIGSEGV on a valid arena address, so the test fails. # test_progs -v -t arena_memcg serial_test_arena_memcg:PASS:child faulted the arena in #8 arena_memcg:OK # without the fix child killed by signal 11 serial_test_arena_memcg:FAIL:child faulted the arena in The page cache must be reclaimable, so the temp file has to live on a disk-backed filesystem, not tmpfs - the same assumption the cgroup selftests make. Signed-off-by: Jiayuan Chen Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260922101831.192102-4-jiayuan.chen@linux.dev --- .../selftests/bpf/prog_tests/arena_memcg.c | 196 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_memcg.c | 23 ++ 2 files changed, 219 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_memcg.c create mode 100644 tools/testing/selftests/bpf/progs/arena_memcg.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_memcg.c b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c new file mode 100644 index 000000000000..c76a7eb2f01d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#define PAGE_SIZE getpagesize() +#endif + +#include "cgroup_helpers.h" +#include "arena_memcg.skel.h" + +#define CG_PATH "/arena_memcg" + +/* Reclaimable page cache the child builds up before it gets capped. */ +#define RECLAIMABLE (128 * 1024 * 1024) +/* Headroom left under memory.max, far less than the arena we fault in. */ +#define HEADROOM (8 * 1024 * 1024) +/* Arena to fault in; it only fits by reclaiming the page cache. */ +#define ARENA_FAULT (64 * 1024 * 1024) +/* Child exit code for "this environment cannot host the test". */ +#define CHILD_UNSUPPORTED 9 + +static void dump_memcg(void) +{ + char buf[512]; + + if (!read_cgroup_file(CG_PATH, "memory.max", buf, sizeof(buf))) + fprintf(stderr, "memory.max: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.peak", buf, sizeof(buf))) + fprintf(stderr, "memory.peak: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.events", buf, sizeof(buf))) + fprintf(stderr, "memory.events:\n%s", buf); +} + +/* + * Fill the page cache with @size bytes of clean, reclaimable pages by + * reading a sparse temp file, the way the cgroup selftests do. Returns the + * fd, which must stay open: closing it drops the cache. Returns -EOPNOTSUPP + * if the working directory cannot back such a file, -1 on error. + */ +static int alloc_pagecache(size_t size) +{ + struct statfs stfs; + char buf[4096]; + size_t off; + int fd; + + fd = open(".", O_TMPFILE | O_RDWR | O_EXCL, 0600); + if (fd < 0) + return errno == EOPNOTSUPP ? -EOPNOTSUPP : -1; + /* tmpfs hands out shmem pages, which are not reclaimable without swap */ + if (fstatfs(fd, &stfs) || stfs.f_type == TMPFS_MAGIC) { + close(fd); + return -EOPNOTSUPP; + } + if (ftruncate(fd, size)) + goto err; + for (off = 0; off < size; off += sizeof(buf)) + if (read(fd, buf, sizeof(buf)) < 0) + goto err; + return fd; +err: + close(fd); + return -1; +} + +void serial_test_arena_memcg(void) +{ + int cgroup_fd = -1, status, err; + const long ps = PAGE_SIZE; + char buf[64]; + pid_t pid; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + goto out; + + cgroup_fd = create_and_get_cgroup(CG_PATH); + if (!ASSERT_OK_FD(cgroup_fd, "create_and_get_cgroup")) + goto out; + + /* No memory controller -> nothing to test. */ + if (read_cgroup_file(CG_PATH, "memory.current", buf, sizeof(buf))) { + fprintf(stderr, "%s:SKIP:no memory controller or other env error\n", + __func__); + test__skip(); + goto out; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto out; + if (pid == 0) { + struct arena_memcg *cskel; + __u32 i, npages; + char *base; + size_t sz; + long cur; + int fd; + + /* + * Everything runs in the child: the arena vma is VM_DONTCOPY so + * it does not survive fork(), and only the child should be under + * the limit. The work dir belongs to the parent, so use the + * _parent() helpers; errors come back as an exit code, ASSERT_* + * does not reach the parent from here. + */ + + /* Step 1: join the memcg, so what follows is charged to it. */ + snprintf(buf, sizeof(buf), "%d", getpid()); + if (write_cgroup_file_parent(CG_PATH, "cgroup.procs", buf)) + _exit(2); + + /* + * Step 2: load the arena. A map is charged to whoever creates + * it, hence joining first. + */ + cskel = arena_memcg__open_and_load(); + if (!cskel) + _exit(3); + base = bpf_map__initial_value(cskel->maps.arena, &sz); + if (!base) + _exit(4); + npages = ARENA_FAULT / ps; + if (npages > bpf_map__max_entries(cskel->maps.arena)) + _exit(5); + + /* Step 3: make RECLAIMABLE bytes of clean page cache. */ + fd = alloc_pagecache(RECLAIMABLE); + if (fd == -EOPNOTSUPP) + _exit(CHILD_UNSUPPORTED); + if (fd < 0) + _exit(6); + + /* + * Step 4: set memory.max to what we use now plus HEADROOM. The + * page cache is already inside the limit, so only HEADROOM is + * left. + */ + if (read_cgroup_file_parent(CG_PATH, "memory.current", buf, sizeof(buf))) + _exit(7); + cur = strtol(buf, NULL, 10); + snprintf(buf, sizeof(buf), "%ld", cur + HEADROOM); + if (write_cgroup_file_parent(CG_PATH, "memory.max", buf)) + _exit(8); + + /* + * Step 5: fault ARENA_FAULT of arena in, much more than + * HEADROOM. Once it hits memory.max every further page has to + * come from reclaiming the page cache. With the fix the + * fault-in reclaims and all of it succeeds; without it the + * allocation cannot reclaim and we die on a valid address. + */ + for (i = 0; i < npages; i++) + base[(size_t)i * ps] = 1; + _exit(0); /* fd deliberately kept open until here */ + } + + if (!ASSERT_EQ(waitpid(pid, &status, 0), pid, "waitpid")) + goto out; + + /* The working directory cannot hold a reclaimable page cache. */ + if (WIFEXITED(status) && WEXITSTATUS(status) == CHILD_UNSUPPORTED) { + fprintf(stderr, "%s:SKIP:no disk-backed O_TMPFILE in cwd\n", __func__); + test__skip(); + goto out; + } + + /* A non-zero exit means the child failed to set up; the code says where. */ + if (WIFEXITED(status) && WEXITSTATUS(status)) { + ASSERT_OK(WEXITSTATUS(status), "child setup"); + goto out; + } + + /* + * With the fix the arena fault-in reclaims the page cache and every + * fault succeeds, so the child exits 0. Without it the allocation + * cannot reclaim, fails once the headroom is used up, and the child + * dies with SIGSEGV on a valid arena address. + */ + if (!ASSERT_TRUE(WIFEXITED(status) && !WEXITSTATUS(status), + "child faulted the arena in")) { + if (WIFSIGNALED(status)) + fprintf(stderr, "child killed by signal %d\n", WTERMSIG(status)); + dump_memcg(); + } +out: + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/progs/arena_memcg.c b/tools/testing/selftests/bpf/progs/arena_memcg.c new file mode 100644 index 000000000000..aff73757e794 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_memcg.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* + * Number of pages. Must cover ARENA_FAULT on the smallest page size + * (64M/4K = 16384) yet stay under the 4G arena limit on 64K pages + * (50000*64K = 3.2G). + */ + __uint(max_entries, 50000); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +char _license[] SEC("license") = "GPL"; From 91f8613d95ad8cd99d8baf094806d1ef98bc6380 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Tue, 22 Sep 2026 19:06:18 +0800 Subject: [PATCH 225/660] bpf: Drop duplicate check_app_limited in tcp_bpf_push Commit c5c37af6ecad9 ("tcp: Convert do_tcp_sendpages() to use MSG_SPLICE_PAGES") moved tcp_rate_check_app_limited() inside do_tcp_sendpages(), turning it into a wrapper around tcp_sendmsg_locked(). Later, commit ebf2e8860eea ("tcp_bpf: Inline do_tcp_sendpages as it's now a wrapper around tcp_sendmsg") inlined the wrapper in tcp_bpf_push() with direct tcp_sendmsg_locked() calls, which perform the check on every path that queues data, but kept the outer tcp_rate_check_app_limited() that was previously needed to cover do_tcp_sendpages(). The outer call is now redundant. The site changed here, tcp_bpf_push(), holds the socket lock and invokes tcp_sendmsg_locked() on every iteration. The early-return paths in tcp_sendmsg_locked() that skip tcp_rate_check_app_limited() - the MSG_ZEROCOPY allocation failure and MSG_FASTOPEN branches - return without queueing any MSG_SPLICE_PAGES data, so there is no functional consequence from omitting the outer check. A potential benefit of this change is that it facilitates future reuse of tcp_bpf_push() for sockmap support in protocols beyond TCP, such as MPTCP. Since tcp_rate_check_app_limited() is TCP-specific while sendmsg_locked() is a generic interface in struct proto_ops, this change allows us to switch to different protocols via sk->sk_socket->ops->sendmsg_locked() without carrying protocol-specific assumptions. Signed-off-by: Geliang Tang Signed-off-by: Alexei Starovoitov Reviewed-by: Jiayuan Chen Link: https://patch.msgid.link/1f7dc605b16fc0590f7cbf5a27d57271926c01ce.1790074764.git.tanggeliang@kylinos.cn --- net/ipv4/tcp_bpf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 2e234d155b5e..d5fcf3ce4861 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -108,7 +108,6 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, off = sge->offset; page = sg_page(sge); - tcp_rate_check_app_limited(sk); retry: msghdr.msg_flags = flags | MSG_SPLICE_PAGES; has_tx_ulp = tls_sw_has_ctx_tx(sk); From 11300f54bd887d15357741ac53db524b710ac484 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Wed, 23 Sep 2026 16:07:17 -0700 Subject: [PATCH 226/660] selftests/bpf: Refactor cb_refs selftests cb_refs matches verifier messages such as "Unreleased reference id=4 alloc_insn=3". The ID is allocated from env->id_gen, so it changes whenever the verifier assigns another ID earlier in the program. In the bpf tree commit 71919742c83c ("bpf: Assign lock identity to callback map values") gives each callback map value an ID, and the nested_cb expectation was changed to id=5. In the bpf-next tree, commit 5dc1549afac9 ("bpf: Consolidate release argument validation") changed leak_prog to expect id=4 from the same leak check. The trees are merged in linux-next, and there leak_prog reports id=5 and the test fails [1]: Expected: Unreleased reference id=4 alloc_insn=3 ... Unreleased reference id=5 alloc_insn=33 The ID is incidental. The test checks that the verifier rejects the leak and which acquisition leaked. Convert the test to test_loader, register the test in prog_tests/verifier.c and drop unnecessary prog_tests/cb_refs.c file. [1] https://github.com/kernel-patches/bpf/actions/runs/35923662147/job/107398076527 Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260923230717.3156345-1-ihor.solodrai@linux.dev --- .../selftests/bpf/prog_tests/cb_refs.c | 48 ------------------- .../selftests/bpf/prog_tests/verifier.c | 3 ++ tools/testing/selftests/bpf/progs/cb_refs.c | 5 ++ 3 files changed, 8 insertions(+), 48 deletions(-) delete mode 100644 tools/testing/selftests/bpf/prog_tests/cb_refs.c diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c deleted file mode 100644 index c32c6dab49bc..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "bpf/libbpf.h" -#include -#include - -#include "cb_refs.skel.h" - -static char log_buf[1024 * 1024]; - -struct { - const char *prog_name; - const char *err_msg; -} cb_refs_tests[] = { - { "underflow_prog", "R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_" }, - { "leak_prog", "Unreleased reference id=4 alloc_insn=3" }, /* alloc_insn=3{2,3} */ - { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ - { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ -}; - -void test_cb_refs(void) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf, - .kernel_log_size = sizeof(log_buf), - .kernel_log_level = 1); - struct bpf_program *prog; - struct cb_refs *skel; - int i; - - for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) { - LIBBPF_OPTS(bpf_test_run_opts, run_opts, - .data_in = &pkt_v4, - .data_size_in = sizeof(pkt_v4), - .repeat = 1, - ); - skel = cb_refs__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load")) - return; - prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name); - bpf_program__set_autoload(prog, true); - if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load")) - bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts); - if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) { - fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg); - fprintf(stderr, "Verifier: %s\n", log_buf); - } - cb_refs__destroy(skel); - } -} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 4f1e1c1cd5ab..8b4be06bb181 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "cb_refs.skel.h" #include "verifier_aggregate_arg.skel.h" #include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" @@ -176,6 +177,8 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_cb_refs(void) { RUN_TESTS(cb_refs); } + void test_verifier_aggregate_arg(void) { RUN_TESTS(verifier_aggregate_arg); } void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c index 5d6fc7f01ebb..d4a691db8a15 100644 --- a/tools/testing/selftests/bpf/progs/cb_refs.c +++ b/tools/testing/selftests/bpf/progs/cb_refs.c @@ -2,6 +2,7 @@ #include #include #include +#include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" struct map_value { @@ -24,6 +25,7 @@ static __noinline int cb1(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int underflow_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -47,6 +49,7 @@ static __always_inline int cb2(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=3") /* alloc_insn=3{0,2,3} */ int leak_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -83,6 +86,7 @@ static __always_inline int cb3(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=2") /* alloc_insn=2{2,4,5} */ int nested_cb(void *ctx) { struct prog_test_ref_kfunc *p; @@ -98,6 +102,7 @@ int nested_cb(void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=1") /* alloc_insn=1{1,2} */ int non_cb_transfer_ref(void *ctx) { struct prog_test_ref_kfunc *p; From 3941710a504a70502f3848145cee200807aae876 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Wed, 23 Sep 2026 16:58:16 +0800 Subject: [PATCH 227/660] bpf: Fix range_tree_clear inconsistency on kmalloc_nolock failure in arena range_tree_clear() pre-allocates the right-half node before modifying the tree, so an allocation failure returns -ENOMEM without altering the range tree. Signed-off-by: Yuan Chen Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260923085817.365930-2-chenyuan_fl@163.com --- kernel/bpf/range_tree.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886f3ff7..15b588377a76 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* + * Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); From b51b058999e0c8ae6fc7b43ef31930e19c768859 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Wed, 23 Sep 2026 16:58:17 +0800 Subject: [PATCH 228/660] bpf: Fix range_tree_set inconsistency on kmalloc_nolock failure in arena range_tree_set() pre-allocates the node needed for a brand-new range before calling range_tree_clear(), so an allocation failure returns -ENOMEM without having modified the tree (previously the overlapping nodes were already removed by range_tree_clear() before the allocation was attempted, permanently losing the cleared sub-ranges). The adjacency lookups done before the clear already return the nodes that the merge below needs: range_tree_clear() frees only the nodes that lie entirely inside [start, last], trims the overlapping ones in place, and the split case cannot happen because of the 'already set' check above. Do the lookup once and drop the unreachable -EFAULT checks. Signed-off-by: Yuan Chen Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260923085817.365930-3-chenyuan_fl@163.com --- kernel/bpf/range_tree.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 15b588377a76..2460e78d6a55 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -199,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -208,20 +209,19 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; - - /* Do we have a left-adjacent range ? */ - left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; - - /* Do we have a right-adjacent range ? */ - right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + goto out_free_new; if (left && right) { /* Combine left and right adjacent ranges */ @@ -241,14 +241,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt) From f2c33a159190c7186bd5f2aac0812663130d20b2 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:26 +0000 Subject: [PATCH 229/660] bpf: Fix infinite loop in check_max_stack_depth() sort_subprogs_topo() rejects recursion via direct calls, but allows cycles that go through ld_imm64 BPF_PSEUDO_FUNC, since the address could be taken for an async callback. The main verifier pass limits the number of frames, but it doesn't follow calls into global functions. So the following program passes both: static int cb(u32 i, void *ctx) { return foo(i); } static int foo(int x) { return bar(x); } int bar(int x) { bpf_loop(1, cb, NULL, 0); return 0; } SEC("socket") int prog(void *ctx) { return cb(0, NULL); } cb -> foo -> bar -> bpf_loop -> cb is unbounded recursion. check_max_stack_depth_subprog() walks this cycle too. None of the functions use stack and the frame counter is reset by every global function, so neither stack size nor frame limit is hit and bpf_prog_load() spins forever. Reject a call into a subprog that is already in the chain of callers. Async callbacks are skipped earlier and are not affected. Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924031042.1690890-2-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a7c9e2d8965d..74cf24ce3007 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5510,6 +5510,27 @@ continue_func: } } + /* + * sort_subprogs_topo() tolerates cycles in the call graph that + * go through the address of a function being taken, since it + * doesn't know what it is taken for. Such cycle is a recursion + * unless it's an async callback, which are skipped above. + * The main verification pass limits the depth of the recursion, + * but it doesn't follow calls of global functions. + */ + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (tmp != sidx) + continue; + verbose(env, "recursive call from %s() to %s()\n", + bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); + bpf_diag_program_structure( + env, i, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This call or callback would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); + return -EINVAL; + } + /* store caller info for after we return from callee */ dinfo[idx].frame = frame; dinfo[idx].ret_insn = i + 1; From 6cec7366b5ceec51d6c65c17ed1086798cea72de Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:27 +0000 Subject: [PATCH 230/660] selftests/bpf: Test recursion through a global function and a callback Add a test for recursion callback -> static func -> global func -> bpf_loop() -> callback that used to hang check_max_stack_depth(). Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-3-alexei.starovoitov@gmail.com --- .../bpf/progs/verifier_global_subprogs.c | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 966f49348787..27fbe54e8795 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -536,4 +536,35 @@ int return_from_void_global(struct __sk_buff *skb) return 0; } +int global_calls_loop(int x); + +static __noinline int static_calls_global(int x) +{ + return global_calls_loop(x); +} + +static __noinline int loop_cb_calls_static(u32 i, void *ctx) +{ + return static_calls_global(i); +} + +__noinline int global_calls_loop(int x) +{ + bpf_loop(1, loop_cb_calls_static, NULL, 0); + return 0; +} + +/* + * loop_cb_calls_static() -> static_calls_global() -> global_calls_loop() -> + * bpf_loop() -> loop_cb_calls_static() is an unbounded recursion that the + * main verification pass can't see, because it doesn't follow calls of global + * functions. None of the functions use stack. + */ +SEC("?raw_tp") +__failure __msg("recursive call from global_calls_loop() to loop_cb_calls_static()") +int recursion_via_global_func_and_callback(const void *ctx) +{ + return loop_cb_calls_static(0, NULL); +} + char _license[] SEC("license") = "GPL"; From fb1f5ac185b76e8e27c1d857dd8b1edc8c9e5a3d Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:28 +0000 Subject: [PATCH 231/660] bpf: Keep functions with address taken when removing dead code ld_imm64 BPF_PSEUDO_FUNC may take the address of a function that nothing calls: the helper call that takes it as a callback is dead code, or the prog only stores or compares the pointer. The function is not seen by the main pass and bpf_opt_remove_dead_code() removes it, while ld_imm64 stays and points to whatever follows. If the function was the last one jit_subprogs() hits: verifier bug: No program to jit at insn 8 (subprog < 0) Otherwise the register gets the address of the next function. Keep the first insn of such function and make it a trap like sanitize_dead_code() does, so the address stays valid and unique. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-4-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 2add8001c3ec..75936842d177 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -619,12 +619,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) } } +/* + * The address of a function might be taken by the code that is not dead, + * while the function itself is dead code, since nothing calls it. Keep the first + * instruction of such function, so that the address remains valid and differs + * from the addresses of other functions. Make it a trap like + * sanitize_dead_code() does. + */ +static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + int i, t; + + for (i = 0; i < insn_cnt; i++) { + if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) + continue; + t = i + insn[i].imm + 1; + if (aux_data[t].seen) + continue; + memcpy(insn + t, &trap, sizeof(trap)); + aux_data[t].zext_dst = false; + aux_data[t].seen = env->pass_cnt; + } +} + int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err; + keep_funcs_with_addr_taken(env); + for (i = 0; i < insn_cnt; i++) { int j; From 373e3fd0bff0e276fd76c86c830cdb16a3457776 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:29 +0000 Subject: [PATCH 232/660] bpf: Prepare static analysis passes for callx instruction BPF_JMP | BPF_CALL | BPF_X (callx) will be an indirect call of static subprog with address in dst_reg. The passes that run before the main verifier pass don't know which subprog callx calls. Teach them to treat callx as a call with unknown callee: - const_fold: callx clobbers R0-R5 like any other call. Otherwise bpf_prune_dead_branches() could rewrite a live conditional jump. - live regs: callx uses R1-R5 and dst_reg, defines R0-R5. - stack liveness: func instances are keyed by (callsite, depth) and cannot describe a callsite with multiple callees. Don't create instances for callees of callx. If any callx argument is derived from fp mark stack of all frames as read at callx insn and keep slots of outer frames alive 'before' the callsite while the callee is verified. Same as for callbacks that are not known statically. The callee is analyzed as standalone instance. - backtracking: treat callx as a call of static subprog, same as BPF_PSEUDO_CALL. callx is still rejected as unknown opcode. No functional change. Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924031042.1690890-5-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 6 ++++++ kernel/bpf/backtrack.c | 20 +++++++++++-------- kernel/bpf/const_fold.c | 3 ++- kernel/bpf/liveness.c | 38 +++++++++++++++++++++++++++++++----- 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 92f528c45605..6380c851ed24 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1089,6 +1089,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 507a366dffa4..4da99dec0818 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -406,15 +406,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { - if (bpf_pseudo_call(insn)) { - int subprog_insn_idx, subprog; + if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) { + int subprog_insn_idx, subprog = -1; - subprog_insn_idx = idx + insn->imm + 1; - subprog = bpf_find_subprog(env, subprog_insn_idx); - if (subprog < 0) - return -EFAULT; + if (bpf_pseudo_call(insn)) { + subprog_insn_idx = idx + insn->imm + 1; + subprog = bpf_find_subprog(env, subprog_insn_idx); + if (subprog < 0) + return -EFAULT; + } - if (bpf_subprog_is_global(env, subprog)) { + /* callx calls static subprogs only */ + if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -536,7 +539,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * never do that. */ from_subprog_call = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) || + bpf_is_callx(&env->prog->insnsi[subseq_idx - 1])); r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 7f1b30059cc8..28dec733e596 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -191,7 +191,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * dst->val = val; break; case BPF_JMP: - if (opcode != BPF_CALL) + /* both 'call imm' and 'callx reg' clobber caller saved registers */ + if (BPF_OP(insn->code) != BPF_CALL) break; process_call: for (r = BPF_REG_0; r <= BPF_REG_5; r++) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 44ecdc5b4ec2..a204b492acc8 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -356,12 +356,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ return 0; } +/* + * Stack slots of outer frames that the callee may read are accounted as read + * by the @callsite insn itself, i.e. they are live before @callsite and may be + * dead after it, when: + * - @callsite calls a callback, which may be called several times; + * - @callsite is callx. Its callee is not known when stack liveness is + * computed, so there is no func instance for it at @callsite. While + * the verifier is in the callee lookup_instance() finds its standalone + * instance that knows nothing about outer frames. What the callee may read + * there is recorded by record_call_access(). + */ +static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite) +{ + return bpf_calls_callback(env, callsite) || + bpf_is_callx(&env->prog->insnsi[callsite]); +} + bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi) { /* * Slot is alive if it is read before q->insn_idx in current func instance, * or if for some outer func instance: - * - alive before callsite if callsite calls callback, otherwise + * - alive before callsite if callsite calls callback or is callx, otherwise * - alive after callsite */ struct live_stack_query *q = &env->liveness->live_stack_query; @@ -394,7 +411,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp /* Get callsite from verifier state, not from instance callchain */ callsite = q->callsites[i]; - alive = bpf_calls_callback(env, callsite) + alive = callee_stack_access_at_callsite(env, callsite) ? is_live_before(instance, callsite, rel, half_spi) : is_live_before(instance, callsite + 1, rel, half_spi); if (alive) @@ -1439,7 +1456,15 @@ static int record_call_access(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) return 0; - if (bpf_get_call_summary(env, insn, &cs)) + if (bpf_is_callx(insn)) + /* + * The callee is not known statically. Assume that all arg + * slots are passed and let record_arg_access() conservatively + * mark the stack of all frames as read if any of them is + * derived from a frame pointer. + */ + arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS; + else if (bpf_get_call_summary(env, insn, &cs)) arg_slot_cnt = cs.arg_slot_cnt; for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { @@ -1533,7 +1558,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, bool has_extra = false; u8 cls = BPF_CLASS(insns[idx].code); bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX || - insns[idx].code == (BPF_JMP | BPF_CALL); + (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL); verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); @@ -1722,7 +1747,7 @@ redo: if (err) goto err_free; - if (insn->code == (BPF_JMP | BPF_CALL)) { + if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) { err = record_call_access(env, instance, at_in[i], idx); if (err) goto err_free; @@ -2202,6 +2227,9 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); def = mask_widen(def); use = mask_widen(use); + /* callx reads the address of the callee from dst_reg */ + if (bpf_is_callx(insn)) + use |= dst; break; default: def = 0; From 7f0f66c069ae06d23710459b45857d4ba2843076 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:30 +0000 Subject: [PATCH 233/660] bpf: Add callx instruction to call bpf subprogs indirectly Introduce BPF_JMP | BPF_CALL | BPF_X (opcode 0x8d) 'callx dst_reg' instruction: indirect call of bpf subprog with address in dst_reg. That's the encoding LLVM emits for calls via function pointer. src_reg, off, imm are reserved and must be zero. dst_reg must be PTR_TO_FUNC produced by ld_imm64 BPF_PSEUDO_FUNC. check_ld_imm() allows it for static subprogs only, so callx cannot call global subprogs or the main prog. Since every callee has its address taken by ld_imm64, add_subprogs() and check_cfg() see all of them before the main pass, and might_sleep, changes_pkt_data, might_throw of the callee are already merged into the subprog that takes the address. reg->subprogno is the callee. Verify callx as a direct call of that static subprog: split check_func_call() into check_static_func_call() that is shared with new check_func_callx(). Different paths through the same callx may call different subprogs. Arithmetic on PTR_TO_FUNC is allowed, so check that the pointer wasn't modified. Allow callx while holding a lock like direct calls of static subprogs. The interpreter doesn't support callx. Set jit_required and add bpf_jit_supports_callx() for JITs to opt in. No JIT does yet, so callx is still rejected. Print it as "callx rN" in the verifier log and xlated dump. Adjust "invalid call insn1" test_verifier test that used opcode 0x8d as unknown opcode. It fails with "R0 !read_ok" now. Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924031042.1690890-6-alexei.starovoitov@gmail.com --- include/linux/filter.h | 1 + kernel/bpf/core.c | 7 + kernel/bpf/disasm.c | 5 +- kernel/bpf/verifier.c | 163 ++++++++++++++---- .../selftests/bpf/verifier/basic_call.c | 2 +- 5 files changed, 145 insertions(+), 33 deletions(-) diff --git a/include/linux/filter.h b/include/linux/filter.h index 422284b4fa96..4f0662e42897 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 227211166dcc..868350977f0c 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1831,6 +1831,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, [BPF_JMP | BPF_JA | BPF_X] = true, + [BPF_JMP | BPF_CALL | BPF_X] = true, [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL @@ -3287,6 +3288,12 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ +bool __weak bpf_jit_supports_callx(void) +{ + return false; +} + bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) { return false; diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 3ce8d74b0e40..36d3228d7745 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (opcode == BPF_CALL) { char tmp[64]; - if (insn->src_reg == BPF_PSEUDO_CALL) { + if (BPF_SRC(insn->code) == BPF_X) { + verbose(cbs->private_data, "(%02x) callx r%d", + insn->code, insn->dst_reg); + } else if (insn->src_reg == BPF_PSEUDO_CALL) { verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 74cf24ce3007..eb014bf6a2e8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6443,6 +6443,21 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +/* there is no support for callx in the interpreter */ +static int require_callx_jit(struct bpf_verifier_env *env) +{ + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use callx\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_callx()) { + verbose(env, "JIT doesn't support callx\n"); + return -EOPNOTSUPP; + } + env->prog->jit_required = true; + return 0; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { @@ -10621,13 +10636,61 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit); -static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - int *insn_idx) +/* + * Call of a static subprog. The callee is verified in the context of + * the caller, hence set up a new frame and continue from the first + * instruction of the callee. + */ +static int check_static_func_call(struct bpf_verifier_env *env, int subprog, + int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; + int err; + + caller = state->frame[state->curframe]; + + /* + * Track caller's total stack arg count (incoming + max outgoing). + * This is needed so the JIT knows how much stack arg space to allocate. + */ + caller_info = &env->subprog_info[caller->subprogno]; + callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); + stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; + if (stack_arg_cnt > caller_info->stack_arg_cnt) + caller_info->stack_arg_cnt = stack_arg_cnt; + + /* + * For regular function entry setup new frame and continue + * from that frame. + */ + err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); + if (err) + return err; + + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + clear_caller_saved_regs(env, caller->regs); + + /* and go analyze first insn of the callee */ + *insn_idx = env->subprog_info[subprog].start - 1; + + if (env->log.level & BPF_LOG_LEVEL) { + verbose(env, "caller:\n"); + print_verifier_state(env, state, caller->frameno, true); + verbose(env, "callee:\n"); + print_verifier_state(env, state, state->curframe, true); + } + + return 0; +} + +static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_func_state *caller; int err, subprog, target_insn; u32 i, nregs; @@ -10716,37 +10779,63 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } - /* - * Track caller's total stack arg count (incoming + max outgoing). - * This is needed so the JIT knows how much stack arg space to allocate. - */ - caller_info = &env->subprog_info[caller->subprogno]; - callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); - stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; - if (stack_arg_cnt > caller_info->stack_arg_cnt) - caller_info->stack_arg_cnt = stack_arg_cnt; + return check_static_func_call(env, subprog, insn_idx); +} - /* for regular function entry setup new frame and continue - * from that frame. - */ - err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); +/* + * callx dst_reg: call a bpf subprog whose address is in dst_reg. + * + * The address of a subprog is loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, which is allowed for static subprogs only. + * Hence all possible callees of callx are discovered by add_subprogs() and + * are reachable in the control flow graph before the main verification pass + * begins. PTR_TO_FUNC register identifies the callee, so from here on callx + * is verified as a direct call of that static subprog. + */ +static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + const char *reason; + int err, subprog; + + err = require_callx_jit(env); if (err) return err; - bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); - clear_caller_saved_regs(env, caller->regs); + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; - /* and go analyze first insn of the callee */ - *insn_idx = env->subprog_info[subprog].start - 1; - - if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "caller:\n"); - print_verifier_state(env, state, caller->frameno, true); - verbose(env, "callee:\n"); - print_verifier_state(env, state, state->curframe, true); + reg = reg_state(env, insn->dst_reg); + if (reg->type != PTR_TO_FUNC) { + verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, + reg_type_str(env, reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s, but callx can only call through the address of a static BPF function.", + insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); + bpf_diag_register_type( + env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, + "Load the address of a static BPF function into the register before callx."); + return -EACCES; } - return 0; + /* + * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address + * of a subprog can be called. + */ + err = check_ptr_off_reg(env, reg, insn->dst_reg); + if (err) + return err; + + /* check_ld_imm() allows to take the address of static subprogs only */ + subprog = reg->subprogno; + err = btf_check_subprog_call(env, subprog, caller->regs); + if (err == -EFAULT) + return err; + + return check_static_func_call(env, subprog, insn_idx); } int map_set_for_each_callback_args(struct bpf_verifier_env *env, @@ -18811,11 +18900,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { - if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock && - insn->imm != BPF_FUNC_kptr_xchg) || - (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - !kfunc_spin_allowed(env, insn->imm, insn->off))) { + /* similar to static subprog calls callx is allowed under a lock */ + if (!bpf_is_callx(insn) && + ((insn->src_reg == BPF_REG_0 && + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || + (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + !kfunc_spin_allowed(env, insn->imm, insn->off)))) { verbose(env, "function calls are not allowed while holding a lock\n"); bpf_diag_ctx_active( @@ -18828,6 +18919,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; + if (bpf_is_callx(insn)) + return check_func_callx(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -19594,6 +19687,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) switch (opcode) { case BPF_CALL: + if (bpf_is_callx(insn)) { + /* callx dst_reg */ + if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { + verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); + return -EINVAL; + } + return 0; + } if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && diff --git a/tools/testing/selftests/bpf/verifier/basic_call.c b/tools/testing/selftests/bpf/verifier/basic_call.c index a8c6ab4c1622..0f93c4551f23 100644 --- a/tools/testing/selftests/bpf/verifier/basic_call.c +++ b/tools/testing/selftests/bpf/verifier/basic_call.c @@ -4,7 +4,7 @@ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0), BPF_EXIT_INSN(), }, - .errstr = "unknown opcode 8d", + .errstr = "R0 !read_ok", .result = REJECT, }, { From b7151a69bc6e1079ac91876c632450f8ee88976e Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:31 +0000 Subject: [PATCH 234/660] bpf: Add callx calls to the call graph The callee of callx is known to the main verifier pass only, but sort_subprogs_topo() (recursion check) and check_max_stack_depth() (stack size, number of frames, tail_call_reachable, private stack) need the call graph. Both treat ld_imm64 BPF_PSEUDO_FUNC as a call from the subprog that takes the address, which is not where callx calls it: main: r1 = foo ll; call bar bar: callx r1 The call chain is main -> bar -> foo, but only main -> bar and main -> foo are seen. Record caller -> callee edge in a bitmap when the main pass processes callx. It explores all feasible paths, so all feasible edges are there. Then: - rerun sort_subprogs_topo() after the main pass with callx edges. Unbounded recursion is already rejected by the frame limit. This one catches bounded recursion via callx, same as direct recursion. The call graph is not context sensitive: foo() that calls bar() via callx and bar() that calls foo() is a recursion even if it never happens in the same call chain. - check_max_stack_depth_subprog() walks all recorded callees at each callx. The same goes for sync callbacks. The address of a callback may be taken in one subprog and passed to a helper in another one: main: r1 = cb ll; call bar bar: r2 = r1; call bpf_loop Only main -> cb was accounted. Record caller -> callback edge in push_callback_call() and walk the edges at helper and kfunc calls that call callbacks too. JITs pass tail call counter to the callee in a register. callx doesn't. Reject tail_call_reachable callees of callx. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-7-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 7 ++ kernel/bpf/verifier.c | 146 +++++++++++++++++++++++++++++------ 2 files changed, 128 insertions(+), 25 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6380c851ed24..931f305fa440 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -965,6 +965,13 @@ struct bpf_verifier_env { struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index eb014bf6a2e8..d00d0ecab17c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3149,12 +3149,50 @@ next: return 0; } +/* + * The callee of callx is known to the main verification pass only, which + * records the 'caller' -> 'callee' edge of the call graph for the checks + * that follow it: absence of recursion and the maximum stack depth. + */ +static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) +{ + u32 cnt = env->subprog_cnt; + + if (!env->callx_edges) { + env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), + GFP_KERNEL_ACCOUNT); + if (!env->callx_edges) + return -ENOMEM; + } + __set_bit(caller * cnt + callee, env->callx_edges); + return 0; +} + +/* + * Return the first subprog with the number >= 'from' that 'caller' calls + * via callx, or -1 when there is none. + */ +static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) +{ + u32 cnt = env->subprog_cnt; + unsigned long bit, end = (caller + 1) * cnt; + + if (!env->callx_edges || from >= cnt) + return -1; + bit = find_next_bit(env->callx_edges, end, caller * cnt + from); + return bit < end ? bit - caller * cnt : -1; +} + /* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). + * + * The callees of callx are not known before the main verification pass. + * When callx is used the sort is repeated after it with the recorded callx + * edges added to the call graph to reject recursion through indirect calls. */ static int sort_subprogs_topo(struct bpf_verifier_env *env) { @@ -3195,12 +3233,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) int idx = insn_postorder[j]; int callee; - if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) + if (bpf_is_callx(&insn[idx])) { + /* find a callee that is not explored yet */ + callee = -1; + do { + callee = next_callx_callee(env, cur, callee + 1); + } while (callee >= 0 && color[callee] == 2); + if (callee < 0) + continue; + } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { + callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); + if (callee < 0) { + ret = -EFAULT; + goto out; + } + } else { continue; - callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); - if (callee < 0) { - ret = -EFAULT; - goto out; } if (color[callee] == 2) continue; @@ -5372,6 +5420,9 @@ struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */ + int callx_insn; /* callx instruction whose callees are being walked */ + int callx_next; /* next callee of callx_insn to walk */ + bool via_callx; /* the subprogram is entered via callx */ }; /* starting from main bpf function walk all instructions of the function @@ -5391,11 +5442,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* no caller idx */ dinfo[idx].caller = -1; + dinfo[idx].via_callx = false; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + dinfo[idx].callx_insn = -1; if (subprog[idx].has_ld_abs) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -5472,11 +5525,10 @@ continue_func: for (; i < subprog_end; i++) { int next_insn, sidx; - if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && + bpf_is_throw_kfunc(insn + i)) { bool err = false; - if (!bpf_is_throw_kfunc(insn + i)) - continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; @@ -5491,23 +5543,41 @@ continue_func: return -EINVAL; } - if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) - continue; - /* remember insn and function to return to */ - - /* find the callee */ - next_insn = i + insn[i].imm + 1; - sidx = bpf_find_subprog(env, next_insn); - if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) - return -EFAULT; - if (subprog[sidx].is_async_cb) { - /* async callbacks don't increase bpf prog stack size unless called directly */ - if (!bpf_pseudo_call(insn + i)) - continue; - if (subprog[sidx].is_exception_cb) { - verbose(env, "insn %d cannot call exception cb directly", i); - return -EINVAL; + if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { + /* + * Walk the callees and the callbacks recorded by + * the main verification pass one by one, returning to + * this insn after each. + */ + if (dinfo[idx].callx_insn != i) { + dinfo[idx].callx_insn = i; + dinfo[idx].callx_next = 0; } + sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); + if (sidx < 0) + continue; + dinfo[idx].callx_next = sidx + 1; + dinfo[idx].ret_insn = i; + next_insn = subprog[sidx].start; + } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { + /* find the callee */ + next_insn = i + insn[i].imm + 1; + sidx = bpf_find_subprog(env, next_insn); + if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) + return -EFAULT; + if (subprog[sidx].is_async_cb) { + /* async callbacks don't increase bpf prog stack size unless called directly */ + if (!bpf_pseudo_call(insn + i)) + continue; + if (subprog[sidx].is_exception_cb) { + verbose(env, "insn %d cannot call exception cb directly", i); + return -EINVAL; + } + } + /* remember insn to return to */ + dinfo[idx].ret_insn = i + 1; + } else { + continue; } /* @@ -5533,10 +5603,10 @@ continue_func: /* store caller info for after we return from callee */ dinfo[idx].frame = frame; - dinfo[idx].ret_insn = i + 1; /* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx; + dinfo[sidx].via_callx = bpf_is_callx(insn + i); i = next_insn; @@ -5570,6 +5640,14 @@ continue_func: verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } + /* + * JITs pass tail call counter in a register that is + * not available when the callee is called via callx. + */ + if (dinfo[tmp].via_callx) { + verbose(env, "tail_calls are not allowed in functions called via callx\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { @@ -10615,6 +10693,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } + /* + * The address of the callback might be taken in another function or + * read from a map, see check_func_ptr_read(). Like with callx it's + * known here what is called. + */ + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -10835,6 +10922,10 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + return check_static_func_call(env, subprog, insn_idx); } @@ -21978,6 +22069,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = do_check_main(env); ret = ret ?: do_check_subprogs(env); + /* reject recursion through the callx edges found by the main pass */ + if (ret == 0 && env->callx_edges) + ret = sort_subprogs_topo(env); + if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); @@ -22120,6 +22215,7 @@ err_free_env: kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + kvfree(env->callx_edges); bpf_diag_free(env); kvfree(env); return ret; From 69a30e5fe5a8d824614ee69b7383efb29098a296 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:32 +0000 Subject: [PATCH 235/660] bpf, x86: Add JIT support for callx Emit indirect call via the register that holds the address of bpf subprog. ld_imm64 BPF_PSEUDO_FUNC is resolved to func[i]->bpf_func by jit_subprogs() already. It's the same address that is passed to helpers as a callback. Add emit_indirect_call() similar to emit_indirect_jump() used by tail calls and gotox: use ITS or retpoline thunks when enabled. Save/restore r9 around the call when private stack is used, like direct calls do. The verifier guarantees that callees of callx are not tail call reachable, so don't pass tail call counter in rax, which may hold the callee address. FineIBT expects indirect callers to go through CFI preamble. Not done yet. Return false from bpf_jit_supports_callx() in this mode, so such progs are rejected at load time. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-8-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 68 +++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d4a980140b48..9fbef7504e51 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -749,6 +749,46 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip) *pprog = prog; } +static void __emit_indirect_call(u8 **pprog, int reg, bool ereg) +{ + u8 *prog = *pprog; + + if (ereg) + EMIT1(0x41); + + EMIT2(0xFF, 0xD0 + reg); + + *pprog = prog; +} + +/* call *bpf_reg */ +static int emit_indirect_call(u8 **pprog, int bpf_reg, u8 *ip) +{ + u8 *prog = *pprog; + int reg = reg2hex[bpf_reg]; + bool ereg = is_ereg(bpf_reg); + int err = 0; + + if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { + OPTIMIZER_HIDE_VAR(reg); + err = emit_call(&prog, its_static_thunk(reg + 8*ereg), ip); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + __emit_indirect_call(&prog, reg, ereg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { + OPTIMIZER_HIDE_VAR(reg); + if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) + err = emit_call(&prog, &__x86_indirect_call_thunk_array[reg + 8*ereg], ip); + else + err = emit_call(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip); + } else { + __emit_indirect_call(&prog, reg, ereg); + } + + *pprog = prog; + return err; +} + static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; @@ -2941,6 +2981,24 @@ populate_extable: break; } + /* callx: call of a bpf subprog whose address is in dst_reg */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * The verifier makes sure that callees of callx are + * not tail call reachable, hence unlike a direct call + * of a subprog there is no need to pass + * tail_call_cnt_ptr in rax. + */ + if (priv_frame_ptr) { + push_r9(&prog); + ip += 2; + } + if (emit_indirect_call(&prog, insn->dst_reg, ip)) + return -EINVAL; + if (priv_frame_ptr) + pop_r9(&prog); + break; + case BPF_JMP | BPF_TAIL_CALL: if (imm32) emit_bpf_tail_call_direct(bpf_prog, @@ -4414,6 +4472,16 @@ out_priv_stack: return prog; } +bool bpf_jit_supports_callx(void) +{ + /* + * FineIBT poisons ENDBR at the entry of a JITed function and expects + * indirect callers to go through the CFI preamble instead. + * callx doesn't do that yet. + */ + return cfi_mode != CFI_FINEIBT; +} + bool bpf_jit_supports_kfunc_call(void) { return true; From 11a516144d9a8c79df26042d2b5bfb779ea091e6 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:33 +0000 Subject: [PATCH 236/660] bpf, arm64: Add JIT support for callx Emit callx as BLR to the register with the address of the subprog and move x0 into BPF_REG_0 like after a direct call. That's what a direct call of a subprog out of BL range is JITed into already. bpf functions start with BTI JC. The arguments are in x0-x7 and on the stack. Tail call counter ptr and private stack ptr are in callee saved x26 and x27. Signed-off-by: Alexei Starovoitov Reviewed-by: Puranjay Mohan Link: https://patch.msgid.link/20260924031042.1690890-9-alexei.starovoitov@gmail.com --- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 6c04fee46876..9544b2f483e5 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1789,6 +1789,17 @@ emit_cond_jmp: emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } + /* indirect call of a bpf subprog, dst holds its address */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * It's the same as a direct call of a subprog that is out of + * range of BL: the subprog starts with BTI JC, the arguments + * are in place, and the registers that hold the tail call + * counter and the private stack are callee saved. + */ + emit(A64_BLR(dst), ctx); + emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); + break; /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) @@ -2485,6 +2496,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_callx(void) +{ + return true; +} + static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, int bargs_off, int retval_off, int run_ctx_off, bool save_ret) From 3a03595510dce6e62ebf6d78707d03abad4bfbd3 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:34 +0000 Subject: [PATCH 237/660] bpf: Discover subprogs described by func_info add_subprogs() finds subprogs by scanning insns for calls and ld_imm64 BPF_PSEUDO_FUNC. A function that is called via callx through a pointer in data only, e.g. a vtable, is not referenced by any insn, but it must be known before check_cfg() and the main pass. func_info describes all functions of the program and libbpf provides it for every function it puts into the prog. bpf_prepare_btf_info() reads it before add_subprogs(). Register every function from func_info as a subprog. func_info had to match the subprogs found in insns exactly, so existing programs are not affected. A function that is listed in func_info and not referenced by anything is still rejected by check_cfg() as unreachable. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-10-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d00d0ecab17c..71b8825ffe1d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3023,6 +3023,22 @@ static int add_subprogs(struct bpf_verifier_env *env) return ret; } + /* + * func_info describes all functions of the program. Those that are + * referenced only from data, e.g. from a table of functions to be + * called via callx, are not seen by the loop above. They are possible + * callees that have to be known upfront as well. + */ + if (env->bpf_capable) { + struct bpf_prog_aux *aux = env->prog->aux; + + for (i = 1; i < aux->func_info_cnt; i++) { + ret = add_subprog(env, aux->func_info[i].insn_off); + if (ret < 0) + return ret; + } + } + ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; From a04286f6d828870f2ad6b8eb18191b1b24a00f23 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:35 +0000 Subject: [PATCH 238/660] bpf: Recognize pointers to functions in read-only maps Compilers put pointers to functions into read-only data: tables of functions, struct ops, vtables, where they're mixed with sizes, alignments and other data. Rust vtables are like that. Let callx call through them: r6 = vtable ll r1 = *(u64 *)(r6 + 0) // data r2 = *(u64 *)(r6 + 8) // pointer to a function callx r2 libbpf keeps such data in a frozen read-only array map like .rodata and stores the byte offset of static function within the program into the pointer. The kernel guesses: aligned 64-bit value that is equal to the offset of a static subprog is a pointer to it. If the guess is wrong the program either fails to load, since it uses a pointer as a number, or sees an address instead of the number. Safe either way. Add resolve_func_ptrs() that scans such maps of a program with callx after the maps are resolved and before check_cfg(), so all callees of callx, including the ones referenced by data only, are known before the main pass. The prog must be the only user of the map and it must be allowed to leak pointers (CAP_PERFMON), since it reads function addresses as data. - check_cfg() treats ld_imm64 of such map like ld_imm64 BPF_PSEUDO_FUNC for every function in the map: they're reachable, first insns are prune and jump points, effects are merged into the subprog that refers to the map. - 64-bit load of the pointer is PTR_TO_FUNC. If the offset is variable (tbl[i]) all possible offsets, given bounds and var_off, must be pointers. The verifier forks a state for each. If none of them overlaps with a pointer (tbl[i].data) it's a regular load. - The value of the pointer is not known until JIT, so reject everything that would const-fold its bytes: narrow or misaligned loads, const strings. bpf_compute_const_regs() skips them too. - The map has offsets until JIT, so the load of the pointer sets jit_required like callx does. Otherwise the prog that reads the pointer and passes it to a helper as a callback could run in the interpreter. After JIT jit_subprogs() replaces the offsets in the map with addresses of functions. Other progs that use the map were verified with its old content const-folded, so there must be none. Add 'user' to bpf_map: __add_used_map() records the first prog that uses the map and marks the map as shared when another prog comes. Only the sole user can store addresses into the map. After that no other prog can use it. The prog that failed to load is not a user. libbpf loads it again to get the log. The offsets are adjusted when insns are patched or removed, same as subprog starts. Dead function is removed and the pointer becomes NULL. The prog may still read it, e.g. when the load is hoisted above a branch, but callx of it is dead code too. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-11-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 10 + include/linux/bpf_verifier.h | 29 +++ kernel/bpf/cfg.c | 76 +++++++ kernel/bpf/const_fold.c | 3 + kernel/bpf/core.c | 5 + kernel/bpf/fixups.c | 65 +++++- kernel/bpf/verifier.c | 386 ++++++++++++++++++++++++++++++++++- 7 files changed, 565 insertions(+), 9 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e7c5e203eddd..9525d3ecd923 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -344,8 +344,18 @@ struct bpf_map { s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 931f305fa440..578bccf5cf76 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -786,6 +786,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -965,6 +980,13 @@ struct bpf_verifier_env { struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; /* * Call graph edges created by callx instructions. A bitmap of * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) @@ -1324,6 +1346,13 @@ static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 842c7d1eabcc..0de2f634ef67 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -421,6 +421,76 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING; } +/* + * Return pointers to functions in the read-only map that ld_imm64 instruction + * 't' loads the address of, or of its value, if there are any. + */ +static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt) +{ + struct bpf_insn *insn = &env->prog->insnsi[t]; + + *cnt = 0; + if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn)) + return NULL; + if (insn->src_reg != BPF_PSEUDO_MAP_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_FD && + insn->src_reg != BPF_PSEUDO_MAP_IDX) + return NULL; + + return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt); +} + +/* + * ld_imm64 that loads the address of a map that has pointers to functions + * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one + * function: any of them may be read from the map and called via callx later. + * Treat it as a call of all of them. + */ +static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, + struct bpf_func_ptr *ptrs, u32 cnt) +{ + int *insn_stack = env->cfg.insn_stack; + int *insn_state = env->cfg.insn_state; + int ret, w; + u32 i; + + ret = push_insn(t, t + 2, FALLTHROUGH, env); + if (ret) + return ret; + + /* + * Explore one function at a time like push_insn() does, so that + * DISCOVERED function is a caller of 't' and not the one that waits in + * insn_stack[], which merge_callee_effects() relies on. + */ + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + if (insn_state[w]) + continue; + + if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; + + insn_stack[env->cfg.cur_stack++] = w; + insn_state[w] = DISCOVERED; + return KEEP_EXPLORING; + } + + /* all functions are explored, their effects are known */ + mark_prune_point(env, t); + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + merge_callee_effects(env, t, w); + + /* the same marks as push_insn() leaves on a branch target */ + mark_prune_point(env, w); + mark_jmp_point(env, w); + mark_jump_target(env, w); + } + return DONE_EXPLORING; +} + /* * Instructions that can abnormally return from a subprog (tail_call * upon success, ld_{abs,ind} upon load failure) have a hidden exit @@ -453,11 +523,17 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) static int visit_insn(int t, struct bpf_verifier_env *env) { struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; + struct bpf_func_ptr *ptrs; int ret, off, insn_sz; + u32 cnt; if (bpf_pseudo_func(insn)) return visit_func_call_insn(t, insns, env, true); + ptrs = insn_func_ptrs(env, t, &cnt); + if (ptrs) + return visit_func_ptrs_insn(t, env, ptrs, cnt); + /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 28dec733e596..b1528adbeb79 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * bool is_ldsx = mode == BPF_MEMSX; int off = src->val + insn->off; u64 val = 0; + u32 cnt; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || off < 0 || off + size > map->value_size || + /* so are the addresses of functions that the map points to */ + bpf_map_range_func_ptrs(env, map, off, size, &cnt) || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; break; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 868350977f0c..273f74068068 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3029,6 +3029,11 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, map->ops->map_poke_untrack(map, aux); if (sleepable) atomic64_dec(&map->sleepable_refcnt); + /* + * The program that didn't load is not a user of the map. libbpf + * loads the program again to get the log of the verifier. + */ + cmpxchg(&map->user, (unsigned long)aux, 0); bpf_map_put(map); } } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 75936842d177..053b24deb471 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -361,6 +361,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); + bpf_adjust_func_ptrs(env, off, len); adjust_poke_descs(new_prog, off, len); return new_prog; } @@ -559,6 +560,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (err) return err; + /* before subprogs are adjusted, since it looks at them */ + bpf_adjust_func_ptrs_after_remove(env, off, cnt); + err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err; @@ -1124,6 +1128,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } +/* + * Replace the offsets of functions with their addresses in the maps of + * the program, see resolve_func_ptrs() in verifier.c. The program must be + * the only user of such map. From now on no other program can use it, see + * bpf_map_claim(). + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_prog **func) +{ + unsigned long me = (unsigned long)prog->aux; + struct bpf_func_ptr *ptr; + int i, err, subprog; + u64 addr, new, *slot; + + for (i = 0; i < env->func_ptr_cnt; i++) { + ptr = &env->func_ptrs[i]; + + /* pointers are sorted by map */ + if ((!i || ptr->map != ptr[-1].map) && + cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { + verbose(env, "map '%s' is used by another program\n", ptr->map->name); + return -EBUSY; + } + + /* it's the address of the value of the map whatever the offset is */ + err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); + if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) + return -EFAULT; + slot = (u64 *)(unsigned long)(addr + ptr->map_off); + + /* + * The function that is dead code is removed and the pointer is + * NULL. The program may read it, but callx of it is dead code + * too, see check_func_callx(). + */ + new = 0; + if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { + subprog = bpf_find_subprog(env, ptr->xlated_off); + if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", + ptr->xlated_off)) + return -EFAULT; + new = (unsigned long)func[subprog]->bpf_func; + } + + /* the map is frozen and nothing else uses it */ + if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, + "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) + return -EFAULT; + WRITE_ONCE(*slot, new); + } + return 0; +} + static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1314,6 +1371,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) cond_resched(); } + /* the addresses of all functions are final */ + err = resolve_func_ptrs(env, prog, func); + if (err) + goto out_free; + /* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future @@ -1400,8 +1462,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) struct bpf_prog *prog, *orig_prog; u32 *orig_subprog_starts; + /* all functions that the maps point to are removed as dead code */ if (env->subprog_cnt <= 1) - return 0; + return resolve_func_ptrs(env, env->prog, NULL); prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 71b8825ffe1d..fec5a1ae6a4d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3120,6 +3120,8 @@ static int check_subprogs(struct bpf_verifier_env *env) if (BPF_CLASS(code) == BPF_LD && (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -6028,6 +6030,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, return 0; } +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct bpf_func_ptr *a = _a, *b = _b; + + if (a->map != b->map) + return a->map < b->map ? -1 : 1; + if (a->map_off != b->map_off) + return a->map_off < b->map_off ? -1 : 1; + return 0; +} + +/* Find the first pointer to a function at or after 'off' in the value of 'map' */ +static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) +{ + u32 l = 0, r = env->func_ptr_cnt, m; + struct bpf_func_ptr *p; + + while (l < r) { + m = l + (r - l) / 2; + p = &env->func_ptrs[m]; + if (p->map < map || (p->map == map && p->map_off < off)) + l = m + 1; + else + r = m; + } + return l; +} + +/* + * Return pointers to functions that overlap with 'size' bytes at offset 'off' + * of the value of 'map' and their number in 'cnt'. + */ +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt) +{ + u32 first, last; + + *cnt = 0; + if (!env->func_ptr_cnt || !size) + return NULL; + + /* a pointer that starts up to 7 bytes before 'off' overlaps too */ + first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); + last = func_ptr_lower_bound(env, map, off + size); + if (first >= last) + return NULL; + + *cnt = last - first; + return &env->func_ptrs[first]; +} + +/* Return all pointers to functions in the value of 'map' */ +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt) +{ + return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); +} + +/* instructions [off, off + len) replaced the instruction at 'off' */ +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + u32 i; + + if (len <= 1) + return; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + p->xlated_off += len - 1; + } +} + +/* + * Instructions [off, off + len) are about to be removed. It's called before + * the starts of subprogs are adjusted. A subprog is gone when all of its + * instructions are. Otherwise, e.g. when its first instruction is a nop, + * it starts where the removed instructions did. + */ +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + int subprog; + u32 i; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + if (p->xlated_off >= off + len) { + p->xlated_off -= len; + continue; + } + subprog = bpf_find_subprog(env, p->xlated_off); + if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) + p->xlated_off = off; + else + p->xlated_off = BPF_FUNC_PTR_DELETED; + } +} + #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) @@ -6552,12 +6658,132 @@ static int require_callx_jit(struct bpf_verifier_env *env) return 0; } +static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int regno, int subprog) +{ + mark_reg_known_zero(env, regs, regno); + regs[regno].type = PTR_TO_FUNC; + regs[regno].subprogno = subprog; +} + +/* a read from a table of functions branches into that many states at most */ +#define BPF_MAX_FUNC_PTR_TARGETS 64 +/* and that many pointers at most are next to what is read at variable offset */ +#define BPF_MAX_FUNC_PTR_RANGE 4096 + +/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ +static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) +{ + u64 o = min_off, next; + u32 cnt = 0; + + if (!tnum_in(offs, tnum_const(o))) { + next = tnum_step(offs, o); + if (next <= o) + return 0; + o = next; + } + while (o <= max_off && cnt < limit) { + cnt++; + next = tnum_step(offs, o); + if (next <= o) + break; + o = next; + } + return cnt; +} + +/* + * A read from a frozen read-only map that has pointers to functions, see + * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read + * is possible at, e.g. var_off is the stride of the elements of an array. + * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable + * and only pointers can be read, which is how an element of a table of + * functions is loaded, the verification continues with each of them. Other + * reads that overlap with a pointer are rejected, because their result is not + * known until the program is jitted. + * + * Return -ENOENT if there are no pointers to functions in the bytes that are read. + */ +static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int size, int value_regno) +{ + u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; + struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_map *map = reg->map_ptr; + struct bpf_verifier_state *branch; + int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; + struct bpf_func_ptr *ptrs; + u32 i, cnt, n = 0; + u64 o, lo, hi; + int err; + + ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); + if (!ptrs) + return -ENOENT; + if (cnt > BPF_MAX_FUNC_PTR_RANGE) + goto overlap; + + for (i = 0; i < cnt; i++) { + /* the offsets at which the read overlaps with the pointer */ + lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; + lo = max(lo, min_off); + hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); + for (o = lo; o <= hi; o++) { + if (!tnum_in(offs, tnum_const(o))) + continue; + if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) + goto overlap; + if (n == BPF_MAX_FUNC_PTR_TARGETS) { + verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", + map->name, BPF_MAX_FUNC_PTR_TARGETS); + return -E2BIG; + } + subprog = bpf_find_subprog(env, ptrs[i].xlated_off); + if (verifier_bug_if(subprog <= 0, env, + "no function at insn %u for map '%s' offset %u", + ptrs[i].xlated_off, map->name, ptrs[i].map_off)) + return -EFAULT; + targets[n++] = subprog; + } + } + /* pointers are next to what is read, e.g. the data of an array of vtables */ + if (!n) + return -ENOENT; + /* either every offset that the read is possible at is a pointer or none */ + if (count_read_offsets(offs, min_off, max_off, n + 1) != n) + goto overlap; + + /* the map has the offset of the function until the program is jitted */ + err = require_callx_jit(env); + if (err) + return err; + + for (i = 0; i < n - 1; i++) { + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, + env->cur_state->speculative); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, + targets[i]); + } + mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); + return 0; + +overlap: + verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", + size, min_off, max_off, map->name); + return -EACCES; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); int size = bpf_size_to_bytes(bpf_size); struct bpf_map *map = reg->map_ptr; + int err; switch (map->map_type) { case BPF_MAP_TYPE_INSN_ARRAY: @@ -6575,13 +6801,18 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state break; } + if (env->func_ptr_cnt) { + err = check_func_ptr_read(env, reg, off, size, value_regno); + if (err != -ENOENT) + return err; + } + /* If map is read-only, track its contents as scalars. */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr) { int map_off = off + reg->var_off.value; u64 val = 0; - int err; err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) @@ -8893,6 +9124,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env, int map_off; u64 map_addr; char *str_ptr; + u32 cnt; if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL; @@ -8942,6 +9174,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env, verbose(env, "string is not zero-terminated\n"); return -EINVAL; } + /* the bytes of a pointer to a function are not known until the program is jitted */ + if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { + verbose(env, "string overlaps with a pointer to a function\n"); + return -EACCES; + } return 0; } @@ -10888,12 +11125,13 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* * callx dst_reg: call a bpf subprog whose address is in dst_reg. * - * The address of a subprog is loaded into a register by ld_imm64 with - * src_reg == BPF_PSEUDO_FUNC, which is allowed for static subprogs only. - * Hence all possible callees of callx are discovered by add_subprogs() and - * are reachable in the control flow graph before the main verification pass - * begins. PTR_TO_FUNC register identifies the callee, so from here on callx - * is verified as a direct call of that static subprog. + * The address of a subprog is either loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see + * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all + * possible callees of callx are discovered by add_subprogs() and are reachable + * in the control flow graph before the main verification pass begins. + * PTR_TO_FUNC register identifies the callee, so from here on callx is verified + * as a direct call of that static subprog. */ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx) @@ -10932,7 +11170,7 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - /* check_ld_imm() allows to take the address of static subprogs only */ + /* PTR_TO_FUNC is a pointer to a static subprog */ subprog = reg->subprogno; err = btf_check_subprog_call(env, subprog, caller->regs); if (err == -EFAULT) @@ -19603,6 +19841,30 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return 0; } +/* + * Keep track of whether the map is used by one program only. Such program may + * store the addresses of its functions into the map when it's frozen, see + * resolve_func_ptrs(), since nothing else relies on what the map has. After + * that the map is not available to other programs. + */ +static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) +{ + unsigned long me = (unsigned long)env->prog->aux, old; + + for (;;) { + old = READ_ONCE(map->user); + if (old == me || old == BPF_MAP_USER_MANY) + return 0; + if (old & BPF_MAP_USER_PATCHED) { + verbose(env, "map '%s' has addresses of functions of another program\n", + map->name); + return -EBUSY; + } + if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) + return 0; + } +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -19640,6 +19902,10 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; + err = bpf_map_claim(env, map); + if (err) + return err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { err = bpf_insn_array_init(map, env->prog); if (err) { @@ -20056,6 +20322,104 @@ next_insn: return 0; } +/* that many pointers to functions are recognized in the maps of a program at most */ +#define BPF_MAX_FUNC_PTRS (1 << 20) + +static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, + u32 xlated_off) +{ + struct bpf_func_ptr *ptrs; + + if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { + verbose(env, "too many pointers to functions in maps, the limit is %d\n", + BPF_MAX_FUNC_PTRS); + return -E2BIG; + } + + /* grow by doubling, the array is sorted and searched later */ + if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { + ptrs = kvrealloc(env->func_ptrs, + array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), + GFP_KERNEL_ACCOUNT); + if (!ptrs) + return -ENOMEM; + env->func_ptrs = ptrs; + } + env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ + .map = map, + .map_off = map_off, + .orig_off = xlated_off, + .xlated_off = xlated_off, + }; + return 0; +} + +/* + * Compilers put pointers to functions into read-only data: tables of functions, + * structures of operations, vtables, where they are mixed with other data. + * The loader stores such data in a frozen read-only array map and resolves + * a pointer to a static function to the offset in bytes of its first + * instruction in the program: the address of the function in the program. + * + * Find 64-bit values that look like that in the maps of a program that uses + * callx. It's a guess. When the value is not a pointer, the program either + * fails to load, because it does with a pointer what can be done with + * a number only, or it sees the address of a function instead of the number. + * It's known before the control flow graph of the program is built and the main + * verification pass begins which functions may be called via callx. + * + * When the program is jitted the offsets are replaced with the addresses of + * the functions in the map itself, see jit_subprogs(). Hence the program has to + * be the only user of the map, see bpf_map_claim(): nothing else may rely on + * what the map had. + * + * The program reads the addresses of its functions from there like any other + * data, so it has to be allowed to leak pointers. + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env) +{ + int insn_cnt = env->prog->len; + int i, err, subprog; + struct bpf_map *map; + u64 addr, val; + u32 off; + + if (!env->has_callx || !env->allow_ptr_leaks) + return 0; + + for (i = 0; i < env->used_map_cnt; i++) { + map = env->used_maps[i]; + /* coincidences in maps that are shared with other programs don't matter */ + if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) + continue; + if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || + !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || + !IS_ERR_OR_NULL(map->record)) + continue; + if (map->ops->map_direct_value_addr(map, &addr, 0)) + continue; + + for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { + if (!(off % SZ_64K)) + cond_resched(); + val = *(u64 *)(unsigned long)(addr + off); + if (!val || val % sizeof(struct bpf_insn) || + val / sizeof(struct bpf_insn) >= insn_cnt) + continue; + subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); + if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) + continue; + err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); + if (err) + return err; + } + } + if (env->func_ptr_cnt) + sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), + cmp_func_ptrs, NULL); + return 0; +} + /* drop refcnt of maps used by the rejected program */ static void release_maps(struct bpf_verifier_env *env) { @@ -22026,6 +22390,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Find pointers to functions in the read-only maps of the program. */ + ret = resolve_func_ptrs(env); + if (ret < 0) + goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ ret = add_kfuncs(env); if (ret < 0) @@ -22232,6 +22601,7 @@ err_free_env: kvfree(env->succ); kvfree(env->gotox_tmp_buf); kvfree(env->callx_edges); + kvfree(env->func_ptrs); bpf_diag_free(env); kvfree(env); return ret; From 296fd47827540d80fe8940bdadb379cd896b7e71 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:36 +0000 Subject: [PATCH 239/660] libbpf: Support pointers to static functions in data when linking A pointer to a static function in data, e.g. static int (* const handlers[])(void *ctx) = { foo, bar }; is R_BPF_64_ABS64 relocation against .text section symbol with the offset of the function stored in place. The linker rejects it: relocation against STT_SECTION in non-exec section is not supported! so an object with a table of static functions cannot go through 'bpftool gen object', which all selftests do. Functions move by the offset of input .text in the output section. Add it to the pointer. Swap bytes if the object is of foreign endianness, since data sections are kept in the byte order of the object. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-12-alexei.starovoitov@gmail.com --- tools/lib/bpf/linker.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 78f92c39290a..53f64a1a1f25 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -2274,6 +2274,24 @@ static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *ob insn->imm += sec->dst_off / sizeof(struct bpf_insn); else insn->imm += sec->dst_off; + } else if (sym_type == R_BPF_64_ABS64 && + (sec->shdr->sh_flags & SHF_EXECINSTR)) { + /* + * A pointer to a static function in a data section, + * which is stored in place as an offset of the + * function in its section. Data sections are kept + * in the byte order of the object. + */ + void *ptr = dst_linked_sec->raw_data + dst_rel->r_offset; + __u64 off; + + memcpy(&off, ptr, sizeof(off)); + if (linker->swapped_endian) + off = bswap_64(off); + off += sec->dst_off; + if (linker->swapped_endian) + off = bswap_64(off); + memcpy(ptr, &off, sizeof(off)); } else { pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); return -EINVAL; From b223044a68d5ac1ef6da344905a98a4d7e2fe5aa Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:37 +0000 Subject: [PATCH 240/660] libbpf: Resolve pointers to functions in read-only data The kernel recognizes a pointer to a function in frozen read-only array map by value: byte offset of a static function in the program. Make it work for tables of functions, struct ops, vtables in .rodata: static const struct shape_ops square_ops = { 1, area, 10, perimeter }; ... return ops->area(x) * ops->scale; is compiled into r3 = *(u64 *)(r1 + 8) callx r3 where 'area' in .rodata is R_BPF_64_ABS64 relocation against .text. - Collect relocations in .rodata* that are aligned pointers to functions. Relocations in data were ignored. The rest still are. - Append all static functions that the section points to, since any of them can be called, if the prog refers to the section and has callx. callx cannot call global functions. Don't pull them in. Such pointer is NULL. - Functions have different offsets in different progs, so create a copy of the map per prog and store the offsets there. The kernel replaces them with addresses at load and the prog must be the only user of the map. The map of the section itself is left as-is. - Progs without callx cannot call them and the kernel doesn't look for pointers in their data. They keep using the map of the section and nothing is appended to them. - The kernel treats any aligned u64 that matches an offset of a static function as a pointer. Warn if data has one that isn't. Light skeleton is not supported yet. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-13-alexei.starovoitov@gmail.com --- tools/lib/bpf/libbpf.c | 381 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 377 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cd1ea1bb53cb..fc9b7540aca6 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -497,6 +497,8 @@ struct bpf_program { bool autoload; bool autoattach; bool sym_global; + /* the program or a function that it calls has callx */ + bool has_callx; bool mark_btf_static; enum bpf_prog_type type; enum bpf_attach_type expected_attach_type; @@ -601,6 +603,9 @@ struct bpf_map { bool autoattach; __u64 map_extra; struct bpf_program *excl_prog; + /* pointers to functions in the data of an internal map, see obj->func_ptrs */ + struct func_ptr *func_ptrs; + size_t func_ptr_cnt; }; enum extern_type { @@ -780,6 +785,29 @@ struct bpf_object { } *jumptable_maps; size_t jumptable_map_cnt; + /* + * Pointers to functions found in read-only data sections: tables of + * functions, structures of operations, vtables. Sorted by section + * and offset. + */ + struct func_ptr { + int sec_idx; /* ELF section that contains the pointer */ + size_t sec_off; /* offset of the pointer in the section */ + size_t text_off; /* offset of the function in .text section */ + } *func_ptrs; + size_t func_ptr_cnt; + + /* + * Read-only data with pointers to functions is different for every + * program that uses it, because so are the offsets of the functions. + */ + struct { + struct bpf_program *prog; + int map_idx; + int fd; + } *func_ptr_maps; + size_t func_ptr_map_cnt; + struct kern_feature_cache *feat_cache; char *token_path; int token_fd; @@ -845,6 +873,16 @@ static bool insn_is_pseudo_func(struct bpf_insn *insn) return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } +static bool prog_has_callx(const struct bpf_program *prog) +{ + size_t i; + + for (i = 0; i < prog->insns_cnt; i++) + if (prog->insns[i].code == (BPF_JMP | BPF_CALL | BPF_X)) + return true; + return false; +} + static int bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, const char *name, size_t sec_idx, const char *sec_name, @@ -4068,8 +4106,14 @@ static int bpf_object__elf_collect(struct bpf_object *obj) targ_sec_idx >= obj->efile.sec_cnt) return -LIBBPF_ERRNO__FORMAT; - /* Only do relo for section with exec instructions */ + /* + * Only do relo for section with exec instructions, + * struct_ops, maps, and read-only data that might + * have pointers to functions. + */ if (!section_have_execinstr(obj, targ_sec_idx) && + strcmp(name, ".rel" RODATA_SEC) && + !str_has_pfx(name, ".rel" RODATA_SEC ".") && strcmp(name, ".rel" STRUCT_OPS_SEC) && strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && @@ -6471,6 +6515,136 @@ err_close: return err; } +/* + * The kernel recognizes a pointer to a function in a frozen read-only map by + * its value: the offset in bytes of the function in the program. It makes + * callx work for tables of functions, structures of operations and vtables, + * where pointers are mixed with other data. Functions have different offsets + * in different programs, so create a copy of the map for the program. + * The kernel replaces the offsets with the addresses of the functions when it + * loads the program, which has to be the only user of the map. + */ +static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, int map_idx) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + struct bpf_map *map = &obj->maps[map_idx]; + __u32 value_size = map->def.value_size; + size_t i, j, cnt, sec_insn_off; + struct func_ptr *ptrs; + int map_fd, err, zero = 0; + __u64 val; + void *data, *tmp; + + for (i = 0; i < obj->func_ptr_map_cnt; i++) + if (obj->func_ptr_maps[i].prog == prog && + obj->func_ptr_maps[i].map_idx == map_idx) + return obj->func_ptr_maps[i].fd; + + if (obj->gen_loader) { + pr_warn("prog '%s': map '%s': pointers to functions in data are not supported by light skeleton\n", + prog->name, map->name); + return -ENOTSUP; + } + + /* the map is not created yet, what it's going to have is in mmaped */ + if (!map->mmaped) + return -EINVAL; + + data = malloc(value_size); + if (!data) + return -ENOMEM; + memcpy(data, map->mmaped, value_size); + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + if (ptrs[i].sec_off + sizeof(val) > value_size) { + err = -LIBBPF_ERRNO__FORMAT; + goto err_free; + } + /* + * Static functions were appended by bpf_object__append_func_ptrs_code(). + * A global function is in the program only if the code refers to it. + */ + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + for (j = 0; j < prog->subprog_cnt; j++) + if (prog->subprogs[j].sec_insn_off == sec_insn_off) + break; + if (j == prog->subprog_cnt) { + pr_debug("prog '%s': map '%s': no function for the pointer at offset %zu, it's NULL\n", + prog->name, map->name, ptrs[i].sec_off); + val = 0; + } else { + val = (__u64)prog->subprogs[j].sub_insn_off * BPF_INSN_SZ; + } + memcpy(data + ptrs[i].sec_off, &val, sizeof(val)); + } + + /* + * The kernel takes any aligned 64-bit value that is equal to the offset + * of a function for a pointer. Tell when it's going to get it wrong. + */ + for (i = 0, j = 0; i + sizeof(val) <= value_size; i += sizeof(val)) { + __u32 k; + + while (j < cnt && ptrs[j].sec_off < i) + j++; + if (j < cnt && ptrs[j].sec_off == i) + continue; + memcpy(&val, data + i, sizeof(val)); + if (!val || val % BPF_INSN_SZ) + continue; + for (k = 0; k < prog->subprog_cnt; k++) { + if ((__u64)prog->subprogs[k].sub_insn_off * BPF_INSN_SZ != val) + continue; + pr_warn("prog '%s': map '%s': value %llu at offset %zu is the offset of a function, the kernel will treat it as a pointer to it\n", + prog->name, map->name, (unsigned long long)val, i); + break; + } + } + + opts.token_fd = obj->token_fd; + if (obj->token_fd) + opts.map_flags |= BPF_F_TOKEN_FD; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, map->name, sizeof(int), value_size, 1, &opts); + if (map_fd < 0) { + err = map_fd; + goto err_free; + } + + err = bpf_map_update_elem(map_fd, &zero, data, 0); + if (!err) + err = bpf_map_freeze(map_fd); + if (err) { + err = -errno; + goto err_close; + } + + tmp = libbpf_reallocarray(obj->func_ptr_maps, obj->func_ptr_map_cnt + 1, + sizeof(*obj->func_ptr_maps)); + if (!tmp) { + err = -ENOMEM; + goto err_close; + } + obj->func_ptr_maps = tmp; + obj->func_ptr_maps[obj->func_ptr_map_cnt].prog = prog; + obj->func_ptr_maps[obj->func_ptr_map_cnt].map_idx = map_idx; + obj->func_ptr_maps[obj->func_ptr_map_cnt].fd = map_fd; + obj->func_ptr_map_cnt++; + + pr_debug("prog '%s': created a copy of map '%s' with %zu pointers to functions\n", + prog->name, map->name, cnt); + free(data); + return map_fd; + +err_close: + close(map_fd); +err_free: + free(data); + return err; +} + /* Relocate data references within program code: * - map references; * - global variable references; @@ -6508,7 +6682,19 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) if (relo->map_idx == obj->arena_map_idx) insn[1].imm += obj->arena_data_off; - if (obj->gen_loader) { + if (map->autocreate && map->func_ptr_cnt && prog->has_callx) { + int map_fd; + + /* the program gets its own map with pointers to its functions */ + map_fd = create_func_ptr_map(obj, prog, relo->map_idx); + if (map_fd < 0) { + pr_warn("prog '%s': relo #%d: can't create a copy of map '%s' with pointers to functions\n", + prog->name, i, map->name); + return map_fd; + } + insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; + insn[0].imm = map_fd; + } else if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; insn[0].imm = relo->map_idx; } else if (map->autocreate) { @@ -6944,6 +7130,75 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, return 0; } +/* Append to the main program all functions that the data of the map points to */ +static int +bpf_object__append_func_ptrs_code(struct bpf_object *obj, struct bpf_program *main_prog, + const struct bpf_map *map) +{ + struct bpf_program *subprog; + size_t i, cnt, sec_insn_off; + struct func_ptr *ptrs; + int err; + + ptrs = map->func_ptrs; + cnt = map->func_ptr_cnt; + for (i = 0; i < cnt; i++) { + sec_insn_off = ptrs[i].text_off / BPF_INSN_SZ; + subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sec_insn_off); + if (!subprog || subprog->sec_insn_off != sec_insn_off) { + pr_warn("prog '%s': map '%s': no function at .text+%zu for the pointer at offset %zu\n", + main_prog->name, map->name, ptrs[i].text_off, ptrs[i].sec_off); + return -LIBBPF_ERRNO__RELOC; + } + + /* + * callx can't call global functions. Don't add one to the + * program only because the data points to it. + */ + if (subprog->sym_global) + continue; + + /* see the comment in bpf_object__reloc_code() */ + if (subprog->sub_insn_off == 0) { + err = bpf_object__append_subprog_code(obj, main_prog, subprog); + if (err) + return err; + err = bpf_object__reloc_code(obj, main_prog, subprog); + if (err) + return err; + } + } + return 0; +} + +/* + * The program that has callx might call any function that the data it refers + * to points to. Append them. Programs that don't have callx can't call them + * and the kernel doesn't look for pointers to functions in their data. + */ +static int +bpf_object__append_func_ptrs(struct bpf_object *obj, struct bpf_program *prog) +{ + size_t i; + int err; + + prog->has_callx = prog_has_callx(prog); + if (!prog->has_callx) + return 0; + + /* relocations of the functions that are appended are appended too */ + for (i = 0; i < prog->nr_reloc; i++) { + struct reloc_desc *relo = &prog->reloc_desc[i]; + + if (relo->type != RELO_DATA || !obj->maps[relo->map_idx].func_ptr_cnt) + continue; + err = bpf_object__append_func_ptrs_code(obj, prog, &obj->maps[relo->map_idx]); + if (err) + return err; + } + return 0; +} + /* * Relocate sub-program calls. * @@ -7046,7 +7301,7 @@ bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) if (err) return err; - return 0; + return bpf_object__append_func_ptrs(obj, prog); } static void @@ -7757,6 +8012,98 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } +/* + * Collect pointers to functions in a read-only data section. They are + * R_BPF_64_ABS64 relocations against .text section, where the offset of + * a static function in the section is stored in place. Relocations in data + * sections were ignored before pointers to functions were supported. Those + * that are something else, e.g. pointers to data, still are. + */ +static int bpf_object__collect_rodata_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + size_t sec_idx = shdr->sh_info, sym_idx; + int i, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + struct func_ptr *ptrs; + Elf_Data *scn_data; + Elf64_Sym *sym; + Elf64_Rel *rel; + __u64 addend; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; + scn_data = obj->efile.secs[sec_idx].data; + if (!scn_data) + return -LIBBPF_ERRNO__FORMAT; + + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + if (!rel) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + + sym_idx = ELF64_R_SYM(rel->r_info); + sym = elf_sym_by_idx(obj, sym_idx); + if (!sym) { + pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", + relo_sec_name, sym_idx, i); + return -LIBBPF_ERRNO__FORMAT; + } + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + !sym_is_subprog(sym, obj->efile.text_shndx)) { + pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", + relo_sec_name, i); + continue; + } + + /* the kernel finds aligned pointers only */ + if (rel->r_offset % sizeof(__u64) || rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_debug("sec '%s': relo #%d: unsupported offset 0x%zx, skipping...\n", + relo_sec_name, i, (size_t)rel->r_offset); + continue; + } + + memcpy(&addend, scn_data->d_buf + rel->r_offset, sizeof(addend)); + if (!is_native_endianness(obj)) + addend = bswap_64(addend); + if ((sym->st_value + addend) % BPF_INSN_SZ) { + pr_debug("sec '%s': relo #%d: bad pointer to a function at offset %zu+%llu, skipping...\n", + relo_sec_name, i, (size_t)sym->st_value, + (unsigned long long)addend); + continue; + } + + ptrs = libbpf_reallocarray(obj->func_ptrs, obj->func_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->func_ptrs = ptrs; + + ptrs[obj->func_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->func_ptr_cnt].sec_off = rel->r_offset; + ptrs[obj->func_ptr_cnt].text_off = sym->st_value + addend; + obj->func_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to a function at .text+%zu\n", + relo_sec_name, i, (size_t)rel->r_offset, (size_t)(sym->st_value + addend)); + } + return 0; +} + +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct func_ptr *a = _a; + const struct func_ptr *b = _b; + + if (a->sec_idx != b->sec_idx) + return a->sec_idx < b->sec_idx ? -1 : 1; + if (a->sec_off != b->sec_off) + return a->sec_off < b->sec_off ? -1 : 1; + return 0; +} + static int bpf_object__collect_relos(struct bpf_object *obj) { int i, err; @@ -7779,7 +8126,9 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return -LIBBPF_ERRNO__INTERNAL; } - if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) + if (obj->efile.secs[idx].sec_type == SEC_RODATA) + err = bpf_object__collect_rodata_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) err = bpf_object__collect_map_relos(obj, shdr, data); @@ -7789,6 +8138,25 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return err; } + /* sort by section, so that pointers in the data of a map are next to each other */ + if (obj->func_ptr_cnt) + qsort(obj->func_ptrs, obj->func_ptr_cnt, sizeof(*obj->func_ptrs), cmp_func_ptrs); + + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *map = &obj->maps[i]; + size_t j; + + if (map->libbpf_type != LIBBPF_MAP_RODATA) + continue; + for (j = 0; j < obj->func_ptr_cnt; j++) { + if (obj->func_ptrs[j].sec_idx != map->sec_idx) + continue; + if (!map->func_ptr_cnt) + map->func_ptrs = &obj->func_ptrs[j]; + map->func_ptr_cnt++; + } + } + bpf_object__sort_relos(obj); return 0; } @@ -9749,6 +10117,11 @@ void bpf_object__close(struct bpf_object *obj) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); + for (i = 0; i < obj->func_ptr_map_cnt; i++) + close(obj->func_ptr_maps[i].fd); + zfree(&obj->func_ptr_maps); + zfree(&obj->func_ptrs); + if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) zfree(&obj->btf_module_allowlist[i]); From 6ec77f0a76975648bfc15e83aa89698c3e509133 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:38 +0000 Subject: [PATCH 241/660] libbpf: Treat .data.rel.ro as read-only data PIC code keeps constants with pointers, e.g. vtables, in .data.rel.ro instead of .rodata. Nothing writes there after relocation. libbpf treats every .data* section as writable, while the kernel looks for pointers to functions in frozen read-only maps only. Make .data.rel.ro and .data.rel.ro.* maps read-only and frozen like .rodata and collect pointers to functions there. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-14-alexei.starovoitov@gmail.com --- tools/lib/bpf/libbpf.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fc9b7540aca6..2e0cf7c5ae23 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -546,6 +546,7 @@ struct bpf_struct_ops { #define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" +#define DATA_REL_RO_SEC ".data.rel.ro" #define KCONFIG_SEC ".kconfig" #define KSYMS_SEC ".ksyms" #define STRUCT_OPS_SEC ".struct_ops" @@ -4062,6 +4063,17 @@ static int bpf_object__elf_collect(struct bpf_object *obj) err = bpf_object__add_programs(obj, data, name, idx); if (err) return err; + } else if (strcmp(name, DATA_REL_RO_SEC) == 0 || + str_has_pfx(name, DATA_REL_RO_SEC ".")) { + /* + * Constants with pointers in them, e.g. vtables, + * that position independent code keeps here to + * have them relocated. There is nothing that + * writes to it after that. + */ + sec_desc->sec_type = SEC_RODATA; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, DATA_SEC) == 0 || str_has_pfx(name, DATA_SEC ".")) { sec_desc->sec_type = SEC_DATA; @@ -4114,6 +4126,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!section_have_execinstr(obj, targ_sec_idx) && strcmp(name, ".rel" RODATA_SEC) && !str_has_pfx(name, ".rel" RODATA_SEC ".") && + strcmp(name, ".rel" DATA_REL_RO_SEC) && + !str_has_pfx(name, ".rel" DATA_REL_RO_SEC ".") && strcmp(name, ".rel" STRUCT_OPS_SEC) && strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && From 9f99fc05fc92796c5ea835462c375e9649ad810b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:39 +0000 Subject: [PATCH 242/660] libbpf: Support pointers to functions in read-only data in light skeleton Light skeleton doesn't create maps. The loader prog does at run time. Teach gen_loader to create a copy of .rodata map with offsets of functions for a prog, see create_func_ptr_map(). - Such maps are not maps of the object and are not in the loader ctx. Reserve slots in fd_array after the maps of the object for them and close their fds when the loader is done or fails. The progs hold them. Which progs have callx is not known when the loader is initialized, so reserve for every prog if .text has callx and for the progs that have callx otherwise. - Add bpf_gen__func_ptr_map_create() that emits map create, update, freeze and returns the index in fd_array for ld_imm64 BPF_PSEUDO_MAP_IDX_VALUE. - User space can set .rodata before loading the skeleton, see initial_value in bpf_map_desc. Copy it into the copy of the map as well and store the offsets of functions on top, since user space doesn't have them. - Light skeleton can be generated for a target of another endianness. Store the offsets in the byte order of the object. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-15-alexei.starovoitov@gmail.com --- tools/lib/bpf/bpf_gen_internal.h | 13 ++- tools/lib/bpf/gen_loader.c | 188 +++++++++++++++++++++++-------- tools/lib/bpf/libbpf.c | 66 +++++++++-- 3 files changed, 212 insertions(+), 55 deletions(-) diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 6c5ad6c55e8a..206adf28793d 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -51,9 +51,17 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; + /* + * Maps with pointers to functions, that programs get their own copies + * of, take slots in fd_array after nr_obj_maps maps of the object. + */ + __u32 nr_obj_maps; + __u32 nr_func_ptr_maps; + __u32 max_func_ptr_maps; }; -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps); int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); void bpf_gen__free(struct bpf_gen *gen); void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); @@ -68,6 +76,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen, void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *value, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, bool is_ld64, int kind, int insn_idx); diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index af3a04f161ac..251392aa8b41 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -112,13 +112,23 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) +void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps, + int max_func_ptr_maps) { size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; int i; gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); gen->log_level = log_level; + gen->nr_obj_maps = nr_maps; + gen->max_func_ptr_maps = max_func_ptr_maps; + if (nr_maps + max_func_ptr_maps > MAX_USED_MAPS) { + pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); + gen->error = -E2BIG; + return; + } + /* their fds are closed like the fds of the maps of the object when loading fails */ + nr_maps += max_func_ptr_maps; /* save ctx pointer into R6 */ emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); @@ -385,6 +395,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) return gen->error; } emit_sys_close_stack(gen, stack_off(btf_fd)); + /* programs hold their maps with pointers to functions, nothing else needs them */ + for (i = 0; i < gen->nr_func_ptr_maps; i++) + emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->nr_obj_maps + i)); for (i = 0; i < gen->nr_progs; i++) move_stack2ctx(gen, sizeof(struct bpf_loader_ctx) + @@ -1127,52 +1140,41 @@ void bpf_gen__prog_load(struct bpf_gen *gen, gen->nr_progs++; } -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size, __u64 flags) +/* + * if (map_desc[map_idx].initial_value) { + * if (ctx->flags & BPF_SKEL_KERNEL) + * bpf_probe_read_kernel(value, value_size, initial_value); + * else + * bpf_copy_from_user(value, value_size, initial_value); + * nr_more_insns that the caller emits + * } + */ +static void emit_copy_initial_value(struct bpf_gen *gen, int map_idx, int value, + __u32 value_size, int nr_more_insns) { - int attr_size = offsetofend(union bpf_attr, flags); - int map_update_attr, value, key; - union bpf_attr attr; - int zero = 0; + emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, + sizeof(struct bpf_loader_ctx) + + sizeof(struct bpf_map_desc) * map_idx + + offsetof(struct bpf_map_desc, initial_value))); + emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8 + nr_more_insns)); + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value)); + emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); + emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, + offsetof(struct bpf_loader_ctx, flags))); + emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); + emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); +} - memset(&attr, 0, attr_size); - attr.flags = tgt_endian(flags); +/* Update the element of the map whose fd is in the slot map_idx of fd_array */ +static void emit_map_update_elem(struct bpf_gen *gen, int map_idx, union bpf_attr *attr, + int attr_size, int key, int value, __u32 value_size) +{ + int map_update_attr; - value = add_data(gen, pvalue, value_size); - key = add_data(gen, &zero, sizeof(zero)); - - /* - * if (map_desc[map_idx].initial_value) { - * if (ctx->flags & BPF_SKEL_KERNEL) - * bpf_probe_read_kernel(value, value_size, initial_value); - * else - * bpf_copy_from_user(value, value_size, initial_value); - * } - * - * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value that the program signature - * covers and the kernel verifies at load time. For a signed loader - * (gen_hash) the attested blob value must be authoritative, so skip - * the override and leave the signed value in place. - */ - if (!OPTS_GET(gen->opts, gen_hash, false)) { - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, - sizeof(struct bpf_loader_ctx) + - sizeof(struct bpf_map_desc) * map_idx + - offsetof(struct bpf_map_desc, initial_value))); - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, - 0, 0, 0, value)); - emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, - offsetof(struct bpf_loader_ctx, flags))); - emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); - emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); - emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); - } - - map_update_attr = add_data(gen, &attr, attr_size); + map_update_attr = add_data(gen, attr, attr_size); pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, @@ -1185,6 +1187,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, emit_check_err(gen); } +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, + __u32 value_size, __u64 flags) +{ + int attr_size = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + int value, key; + int zero = 0; + + memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); + + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* + * The runtime initial_value comes from the host-supplied loader + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. + */ + if (!OPTS_GET(gen->opts, gen_hash, false)) + emit_copy_initial_value(gen, map_idx, value, value_size, 0); + + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); +} + void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, int inner_map_idx) { @@ -1214,6 +1243,77 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo emit_check_err(gen); } +/* + * A copy of the read-only data map obj_map_idx for a program, with the offsets + * of its functions in it, see create_func_ptr_map() in libbpf.c. It's not + * a map of the object: it's not in the loader ctx. Its content is the content + * of obj_map_idx, that the host may supply when the skeleton is loaded, with + * ptr_cnt 64-bit ptr_vals at ptr_offs. + * Return the index of the map in fd_array for instructions to refer to. + */ +int bpf_gen__func_ptr_map_create(struct bpf_gen *gen, const char *map_name, int obj_map_idx, + void *pvalue, __u32 value_size, const __u32 *ptr_offs, + const __u64 *ptr_vals, int ptr_cnt) +{ + int attr_size = offsetofend(union bpf_attr, map_extra); + int map_create_attr, map_idx, key, value, zero = 0, i; + union bpf_attr attr; + + if (gen->nr_func_ptr_maps == gen->max_func_ptr_maps) { + gen->error = -EDOM; /* internal bug */ + return 0; + } + map_idx = gen->nr_obj_maps + gen->nr_func_ptr_maps++; + + memset(&attr, 0, attr_size); + attr.map_type = tgt_endian(BPF_MAP_TYPE_ARRAY); + attr.key_size = tgt_endian((__u32)sizeof(int)); + attr.value_size = tgt_endian(value_size); + attr.max_entries = tgt_endian((__u32)1); + attr.map_flags = tgt_endian((__u32)BPF_F_RDONLY_PROG); + if (map_name) + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); + + map_create_attr = add_data(gen, &attr, attr_size); + pr_debug("gen: func_ptr_map_create: %s idx %d value_size %u, attr: off %d size %d\n", + map_name, map_idx, value_size, map_create_attr, attr_size); + emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); + debug_ret(gen, "func_ptr_map_create %s idx %d value_size %d", map_name, map_idx, + value_size); + emit_check_err(gen); + /* remember map_fd in fd_array */ + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, blob_fd_array_off(gen, map_idx))); + emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); + + /* pvalue has the pointers already */ + value = add_data(gen, pvalue, value_size); + key = add_data(gen, &zero, sizeof(zero)); + + /* see bpf_gen__map_update_elem() */ + if (!OPTS_GET(gen->opts, gen_hash, false)) { + /* the jump over these instructions has 16-bit offset */ + if (ptr_cnt > 10000) { + gen->error = -E2BIG; + return 0; + } + emit_copy_initial_value(gen, obj_map_idx, value, value_size, 3 * ptr_cnt); + /* the content that the host supplied doesn't have them */ + for (i = 0; i < ptr_cnt; i++) { + emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, + 0, 0, 0, value + ptr_offs[i])); + emit(gen, BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, ptr_vals[i])); + } + } + + attr_size = offsetofend(union bpf_attr, flags); + memset(&attr, 0, attr_size); + emit_map_update_elem(gen, map_idx, &attr, attr_size, key, value, value_size); + + bpf_gen__map_freeze(gen, map_idx); + return map_idx; +} + void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) { int attr_size = offsetofend(union bpf_attr, map_fd); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 2e0cf7c5ae23..39fc3c5e6bd2 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -6554,12 +6554,6 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, obj->func_ptr_maps[i].map_idx == map_idx) return obj->func_ptr_maps[i].fd; - if (obj->gen_loader) { - pr_warn("prog '%s': map '%s': pointers to functions in data are not supported by light skeleton\n", - prog->name, map->name); - return -ENOTSUP; - } - /* the map is not created yet, what it's going to have is in mmaped */ if (!map->mmaped) return -EINVAL; @@ -6591,6 +6585,9 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, } else { val = (__u64)prog->subprogs[j].sub_insn_off * BPF_INSN_SZ; } + /* light skeleton can be generated for a target of another endianness */ + if (!is_native_endianness(obj)) + val = bswap_64(val); memcpy(data + ptrs[i].sec_off, &val, sizeof(val)); } @@ -6606,6 +6603,8 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, if (j < cnt && ptrs[j].sec_off == i) continue; memcpy(&val, data + i, sizeof(val)); + if (!is_native_endianness(obj)) + val = bswap_64(val); if (!val || val % BPF_INSN_SZ) continue; for (k = 0; k < prog->subprog_cnt; k++) { @@ -6617,6 +6616,30 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, } } + if (obj->gen_loader) { + __u32 *ptr_offs = calloc(cnt, sizeof(*ptr_offs)); + __u64 *ptr_vals = calloc(cnt, sizeof(*ptr_vals)); + + if (!ptr_offs || !ptr_vals) { + free(ptr_offs); + free(ptr_vals); + err = -ENOMEM; + goto err_free; + } + for (i = 0; i < cnt; i++) { + ptr_offs[i] = ptrs[i].sec_off; + memcpy(&ptr_vals[i], data + ptrs[i].sec_off, sizeof(val)); + if (!is_native_endianness(obj)) + ptr_vals[i] = bswap_64(ptr_vals[i]); + } + /* it's an index in fd_array of the loader, not an fd */ + map_fd = bpf_gen__func_ptr_map_create(obj->gen_loader, map->name, map_idx, data, + value_size, ptr_offs, ptr_vals, cnt); + free(ptr_offs); + free(ptr_vals); + goto done; + } + opts.token_fd = obj->token_fd; if (obj->token_fd) opts.map_flags |= BPF_F_TOKEN_FD; @@ -6634,6 +6657,7 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, err = -errno; goto err_close; } +done: tmp = libbpf_reallocarray(obj->func_ptr_maps, obj->func_ptr_map_cnt + 1, sizeof(*obj->func_ptr_maps)); @@ -6653,7 +6677,8 @@ static int create_func_ptr_map(struct bpf_object *obj, struct bpf_program *prog, return map_fd; err_close: - close(map_fd); + if (!obj->gen_loader) + close(map_fd); err_free: free(data); return err; @@ -6706,7 +6731,8 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) prog->name, i, map->name); return map_fd; } - insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; + insn[0].src_reg = obj->gen_loader ? BPF_PSEUDO_MAP_IDX_VALUE : + BPF_PSEUDO_MAP_VALUE; insn[0].imm = map_fd; } else if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; @@ -9590,8 +9616,27 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch * permit cross-endian creation of "light skeleton". */ if (obj->gen_loader) { + int nr_func_ptr_maps = 0, nr_progs = 0, i; + bool text_has_callx = false; + + /* + * Every program that has callx may get a copy of every map with + * pointers to functions. Which functions a program calls is not + * known yet. Any of them may call the ones that have callx. + */ + for (i = 0; i < obj->nr_maps; i++) + if (obj->maps[i].autocreate && obj->maps[i].func_ptr_cnt) + nr_func_ptr_maps++; + for (i = 0; i < obj->nr_programs; i++) + if (prog_is_subprog(obj, &obj->programs[i]) && + prog_has_callx(&obj->programs[i])) + text_has_callx = true; + for (i = 0; i < obj->nr_programs; i++) + if (obj->programs[i].autoload && !prog_is_subprog(obj, &obj->programs[i]) && + (text_has_callx || prog_has_callx(&obj->programs[i]))) + nr_progs++; bpf_gen__init(obj->gen_loader, obj->log_level | extra_log_level, - obj->nr_programs, obj->nr_maps); + obj->nr_programs, obj->nr_maps, nr_func_ptr_maps * nr_progs); } else if (!is_native_endianness(obj)) { pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); return libbpf_err(-LIBBPF_ERRNO__ENDIAN); @@ -10132,7 +10177,8 @@ void bpf_object__close(struct bpf_object *obj) zfree(&obj->jumptable_maps); for (i = 0; i < obj->func_ptr_map_cnt; i++) - close(obj->func_ptr_maps[i].fd); + if (!obj->gen_loader) + close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs); From e9b22d50796d793d7184e10c397fe98a55ab5a99 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:40 +0000 Subject: [PATCH 243/660] selftests/bpf: Add tests for callx Add verifier_callx tests: - calls via r0, callee saved regs, pointer passed as argument, returned from subprog, spilled/filled, different callees on different paths. These are executed to test JIT. - invalid operands: scalar, other pointers, modified and variable PTR_TO_FUNC, global functions, reserved fields, JMP32. - callx under lock. - bounded and unbounded recursion via callx. - stack depth of call chains through callx, including address taken several frames above callx. - tail calls in the callee and the caller of callx. - stack liveness: caller slots read by callee stay alive. - no const propagation across callx. - precision backtracking through callx. - address of a function that is never called, with and without callx. - function pointers in C. callx needs JIT. Add RUN_JITED() to skip the tests when it's off. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-16-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 10 + .../selftests/bpf/progs/verifier_callx.c | 1039 +++++++++++++++++ 2 files changed, 1049 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_callx.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8b4be06bb181..bc96a752b1ae 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -28,6 +28,7 @@ #include "verifier_btf_ctx_access.skel.h" #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" +#include "verifier_callx.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -173,6 +174,14 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +/* for tests of what the interpreter doesn't support */ +#define RUN_JITED(skel) do { \ + if (is_jit_enabled()) \ + RUN(skel); \ + else \ + test__skip(); \ +} while (0) + void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } @@ -199,6 +208,7 @@ void test_verifier_bswap(void) { RUN(verifier_bswap); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } +void test_verifier_callx(void) { RUN_JITED(verifier_callx); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c new file mode 100644 index 000000000000..238fc75ad154 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -0,0 +1,1039 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx: indirect calls of bpf subprogs */ + +#include +#include +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) + +#define CALLX_INSN(DST, SRC, OFF, IMM) \ + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} map_array SEC(".maps"); + +struct val_with_lock { + struct bpf_spin_lock lock; + int cnt; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct val_with_lock); +} map_lock SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} map_prog SEC(".maps"); + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* apply(fn, x) { return fn(x); } */ +__naked __noinline __used +static unsigned long apply(void) +{ + asm volatile ( + "r3 = r1;" + "r1 = r2;" + "callx r3;" + "exit;" + ); +} + +SEC("socket") +__success __retval(6) +__naked void callx_basic(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* callx is printed by the verifier log and xlated dump */ +SEC("socket") +__success __log_level(2) +__msg("(8d) callx r2") +__xlated("callx r2") +__naked void callx_disasm(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* different callees are called by the same callx on different paths */ +SEC("socket") +__success __retval(11) +__naked void callx_two_callees(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r2 = %[add1] ll;" + "if r6 == 0 goto +2;" + "r2 = %[add2] ll;" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +/* pointer to a function is passed as an argument */ +SEC("socket") +__success __retval(45) +__naked void callx_fn_as_arg(void) +{ + asm volatile ( + "r1 = %[add2] ll;" + "r2 = 40;" + "call apply;" + "r6 = r0;" + "r1 = %[add1] ll;" + "r2 = 2;" + "call apply;" + "r0 += r6;" + "exit;" + : + : __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long get_add1(void) +{ + asm volatile ( + "r0 = %[add1] ll;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* pointer to a function is returned from a subprog and called via r0 */ +SEC("socket") +__success __retval(2) +__naked void callx_r0(void) +{ + asm volatile ( + "call get_add1;" + "r1 = 1;" + "callx r0;" + "exit;" + ::: __clobber_all); +} + +/* pointer to a function survives spill/fill */ +SEC("socket") +__success __retval(9) +__naked void callx_spill_fill(void) +{ + asm volatile ( + "r2 = %[add2] ll;" + "*(u64 *)(r10 - 8) = r2;" + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "r9 = *(u64 *)(r10 - 8);" + "callx r9;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long clobber_callee_saved(void) +{ + asm volatile ( + "r6 = 100;" + "r7 = 100;" + "r8 = 100;" + "r9 = 100;" + "r0 = r1;" + "exit;" + ); +} + +/* r6-r9 are preserved across callx */ +SEC("socket") +__success __retval(11) +__naked void callx_callee_saved_regs(void) +{ + asm volatile ( + "r6 = 1;" + "r7 = 2;" + "r8 = 3;" + "r9 = 4;" + "r1 = 1;" + "r2 = %[clobber_callee_saved] ll;" + "callx r2;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + : + : __imm_addr(clobber_callee_saved) + : __clobber_all); +} + +/* r1-r5 are scratched by callx */ +SEC("socket") +__failure __msg("R1 !read_ok") +__naked void callx_scratches_args(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "r0 = r1;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* return value of the callee is tracked */ +SEC("socket") +__success __log_level(2) +__msg("to caller at 4:") +__msg("R0=6") +__naked void callx_retval_is_tracked(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long write42(void) +{ + asm volatile ( + "r2 = 42;" + "*(u64 *)(r1 + 0) = r2;" + "r0 = 0;" + "exit;" + ); +} + +/* callee writes into the stack of the caller */ +SEC("socket") +__success __retval(42) +__naked void callx_callee_writes_caller_stack(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "r2 = %[write42] ll;" + "callx r2;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm_addr(write42) + : __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type scalar, expected func") +__naked void callx_scalar(void) +{ + asm volatile ( + "r1 = 0;" + "callx r1;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R2 !read_ok") +__naked void callx_uninit_reg(void) +{ + asm volatile ( + "callx r2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R10 has type fp, expected func") +__naked void callx_fp(void) +{ + asm volatile ( + "callx r10;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type map_value, expected func") +__naked void callx_map_value(void) +{ + asm volatile ( + "r1 = %[map_array] ll;" + "r2 = r10;" + "r2 += -4;" + "r3 = 0;" + "*(u32 *)(r2 + 0) = r3;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r1 = r0;" + "callx r1;" + "1:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* the address of a subprog can't be modified before the call */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_modified_ptr(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "r2 += 8;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +SEC("socket") +__failure __msg("variable func access var_off=") +__naked void callx_variable_ptr(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r2 = %[add1] ll;" + "r2 += r0;" + "r1 = 5;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1) + : __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* only static subprogs can be called via callx */ +SEC("socket") +__failure __msg("callback function not static") +__naked void callx_global_func(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[global_add3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(global_add3) + : __clobber_all); +} + +#define DEFINE_CALLX_RESERVED_FIELDS_PROG(NAME, SRC_REG, OFF, IMM) \ + SEC("socket") \ + __failure __msg("BPF_CALL|BPF_X uses reserved fields") \ + __naked void callx_reserved_field_ ## NAME(void) \ + { \ + asm volatile ( \ + "r1 = 5;" \ + "r2 = %[add1] ll;" \ + ".8byte %[callx_r2];" \ + "exit;" \ + : \ + : __imm_addr(add1), \ + __imm_insn(callx_r2, CALLX_INSN(BPF_REG_2, (SRC_REG), (OFF), (IMM))) \ + : __clobber_all); \ + } + +DEFINE_CALLX_RESERVED_FIELDS_PROG(src_reg, BPF_REG_1, 0, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(off, BPF_REG_0, 1, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(imm, BPF_REG_0, 0, 1) + +SEC("socket") +__failure __msg("unknown opcode 8e") +__naked void callx_jmp32(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + ".8byte %[callx32_r2];" + "exit;" + : + : __imm_addr(add1), + __imm_insn(callx32_r2, + BPF_RAW_INSN(BPF_JMP32 | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0)) + : __clobber_all); +} + +/* similar to calls of static subprogs callx is allowed under a lock */ +SEC("tc") +__success __retval(3) +__naked void callx_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r1 = 1;" + "r2 = %[add2] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(add2) + : __clobber_all); +} + +/* helpers are still not allowed under a lock in the callee */ +__naked __noinline __used +static unsigned long call_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("tc") +__failure __msg("function calls are not allowed while holding a lock") +__naked void callx_helper_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r2 = %[call_helper] ll;" + "callx r2;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(call_helper) + : __clobber_all); +} + +/* self(fn) { return fn(fn); } */ +__naked __noinline __used +static unsigned long self(void) +{ + asm volatile ( + "callx r1;" + "exit;" + ); +} + +/* unbounded recursion is caught by the main verification pass */ +SEC("socket") +__failure __msg("frames is too deep") +__naked void callx_unbounded_recursion(void) +{ + asm volatile ( + "r1 = %[self] ll;" + "call self;" + "exit;" + : + : __imm_addr(self) + : __clobber_all); +} + +/* countdown(fn, n) { return n ? fn(fn, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long countdown(void) +{ + asm volatile ( + "r0 = 0;" + "if r2 == 0 goto 1f;" + "r2 += -1;" + "callx r1;" + "1:" + "exit;" + ); +} + +/* + * The depth of the recursion is known to the main verification pass, + * but recursive calls are not allowed regardless. + */ +SEC("socket") +__failure __msg("recursive call from countdown() to countdown()") +__naked void callx_bounded_recursion(void) +{ + asm volatile ( + "r1 = %[countdown] ll;" + "r2 = 2;" + "call countdown;" + "exit;" + : + : __imm_addr(countdown) + : __clobber_all); +} + +/* ping(fn1, fn2, n) { return n ? fn2(fn2, fn1, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long ping(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long pong(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +SEC("socket") +__failure __msg("recursive call from") +__naked void callx_mutual_recursion(void) +{ + asm volatile ( + "r1 = %[ping] ll;" + "r2 = %[pong] ll;" + "r3 = 3;" + "call ping;" + "exit;" + : + : __imm_addr(ping), + __imm_addr(pong) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + +/* stack of the callee of callx is accounted */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_stack_depth(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +/* apply_stack_304(fn) { char buf[304]; return fn(); } */ +__naked __noinline __used +static unsigned long apply_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "callx r1;" + "exit;" + ); +} + +/* + * The address of use_stack_304() is taken by the main prog that doesn't + * use stack, but it is called from apply_stack_304(). + */ +SEC("socket") +__failure __msg("combined stack size of 3 calls is") +__naked void callx_stack_depth_nested(void) +{ + asm volatile ( + "r1 = %[use_stack_304] ll;" + "call apply_stack_304;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +SEC("socket") +__success __retval(0) +__naked void callx_stack_depth_ok(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 200) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long do_tail_call(void) +{ + asm volatile ( + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog) + : __clobber_all); +} + +SEC("socket") +__failure __msg("tail_calls are not allowed in functions called via callx") +__naked void callx_tail_call_in_callee(void) +{ + asm volatile ( + "r2 = %[do_tail_call] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(do_tail_call) + : __clobber_all); +} + +/* tail call in the caller of callx is fine */ +SEC("socket") +__success __retval(3) +__naked void callx_tail_call_in_caller(void) +{ + asm volatile ( + "r6 = r1;" + "r1 = 2;" + "r2 = %[add1] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog), + __imm_addr(add1) + : __clobber_all); +} + +/* read_idx(p) { return ((char *)map_value)[*p]; } */ +__naked __noinline __used +static unsigned long read_idx(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_array] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r0 += r6;" + "r0 = *(u8 *)(r0 + 0);" + "1:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* + * Stack slots of the caller that might be read by the callee of callx + * have to be considered alive at the checkpoints before callx and + * inside of the callee. Otherwise the state with fp-8 == 1000 is pruned + * and out of bounds access in read_idx() goes unnoticed. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* in bounds access in read_idx() is fine */ +SEC("socket") +__success __retval(0) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_ok(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* same as above, but the pointer to the stack is passed through one more frame */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_nested(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = %[read_idx] ll;" + "r2 = r10;" + "r2 += -8;" + "call apply;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* registers that are constant before callx are not constant after it */ +SEC("socket") +__success __retval(1) +__naked void callx_clobbers_const_regs(void) +{ + asm volatile ( + "r0 = 0;" + "r1 = 0;" + "r2 = %[add1] ll;" + "callx r2;" + /* dead branch pruning must not assume that r0 is still 0 */ + "if r0 == 0 goto 1f;" + "r0 = 1;" + "exit;" + "1:" + "r0 = 2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* scalar argument passed through callx is tracked precisely */ +__naked __noinline __used +static unsigned long identity(void) +{ + asm volatile ( + "r0 = r1;" + "exit;" + ); +} + +long long vals[] SEC(".data.vals") = {1, 2, 3, 4}; + +SEC("socket") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r0 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r0 stack= before 11: (bf) r0 = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 4: (8d) callx r2") +__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") +__retval(4) +__naked void callx_precision(void) +{ + asm volatile ( + "r2 = %[identity] ll;" + "r6 = 3;" + "r1 = r6;" + "callx r2;" + "r0 *= 8;" + "r1 = %[vals] ll;" + "r1 += r0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm_addr(identity), + __imm_addr(vals) + : __clobber_all); +} + +/* + * The address of a function that is never called is valid. The function is + * the last one in the program. + */ +SEC("socket") +__success __retval(6) +__naked void callx_addr_of_dead_func(void) +{ + asm volatile ( + "r6 = %[add1] ll;" + "r7 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* and when it's followed by another function */ +SEC("socket") +__success __retval(7) +__naked void callx_addr_of_dead_func_first(void) +{ + asm volatile ( + "r7 = %[add1] ll;" + "r6 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* the same without callx */ +SEC("socket") +__success __retval(6) +__naked void call_addr_of_dead_func(void) +{ + asm volatile ( + "r1 = 5;" + "call add1;" + "r7 = %[add2] ll;" + "exit;" + : + : __imm_addr(add2) + : __clobber_all); +} + +/* function pointers in C */ + +typedef int (*op_fn)(int); + +static __noinline int mul3(int x) +{ + return x * 3; +} + +static __noinline int sub7(int x) +{ + return x - 7; +} + +static __noinline int apply_op(op_fn op, int x) +{ + return op(x); +} + +SEC("socket") +__success __retval(36) +int callx_c_fn_as_arg(void *ctx) +{ + /* (5 * 3) + (28 - 7) */ + return apply_op(mul3, 5) + apply_op(sub7, 28); +} + +SEC("socket") +__success __retval(30) +int callx_c_select(void *ctx) +{ + __u32 rnd = bpf_get_prandom_u32() & 1; + op_fn op = rnd ? mul3 : sub7; + int x = rnd ? 10 : 37; + + /* 10 * 3 or 37 - 7 */ + return op(x); +} + +struct ops { + op_fn first; + op_fn second; + int bias; +}; + +static __noinline int run_ops(const struct ops *ops, int x) +{ + return ops->second(ops->first(x)) + ops->bias; +} + +SEC("socket") +__success __retval(100) +int callx_c_ops_on_stack(void *ctx) +{ + struct ops a, b; + + /* avoid an initializer with function pointers in .rodata */ + a.first = mul3; + a.second = sub7; + a.bias = 10; + b.first = sub7; + b.second = mul3; + b.bias = 1; + + /* ((7 * 3) - 7 + 10) + ((32 - 7) * 3 + 1) */ + return run_ops(&a, 7) + run_ops(&b, 32); +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; From b018e998e645c107d742935c1483cd77452e1787 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:41 +0000 Subject: [PATCH 244/660] selftests/bpf: Add tests for callx through pointers in read-only data Add verifier_callx_rodata tests where pointers to functions are in .rodata, libbpf stores offsets there and the kernel recognizes them: - vtable-like data in .rodata and .data.rel.ro, one of two vtables picked at run time, const and variable index, array of structs where the index selects the pointer or the data, address of an element, table without a symbol. These are executed. - data next to a pointer is still a constant. The pointer is not: bpf_prune_dead_branches() must not fold a branch on it. - all functions the index can select are verified. The rest are not. - index that may select non-pointer, partial and misaligned reads, arithmetic on the pointer. - pointer to a global function is not recognized. - prog without callx reads the pointer as a number. - recursion and stack depth via a table, stack depth of a callback that is read from a table. - effects of the functions in a table that call each other. - CAP_BPF without CAP_PERFMON. - C: array of functions, NULL check of an element, struct ops with data, switch that may become a table, packed struct with misaligned pointers in a prog without callx. Add callx_func_ptr_map test that doesn't use libbpf logic: map that another prog uses is not scanned, the prog that failed to load is not a user, the offset in the map is replaced at load, the data is not, no other prog including second instance of the same one can use the map after that. Add callx_rodata_lskel test: two progs with a table of functions in .rodata and one without callx loaded by light skeleton, with .rodata set by user space. callx needs JIT. Skip the tests when it's off. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-17-alexei.starovoitov@gmail.com --- tools/testing/selftests/bpf/Makefile.skel | 2 +- .../bpf/prog_tests/callx_func_ptr_map.c | 202 +++++ .../bpf/prog_tests/callx_rodata_lskel.c | 63 ++ .../selftests/bpf/prog_tests/verifier.c | 2 + .../selftests/bpf/progs/callx_rodata.c | 51 ++ .../bpf/progs/verifier_callx_rodata.c | 846 ++++++++++++++++++ 6 files changed, 1165 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c create mode 100644 tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c create mode 100644 tools/testing/selftests/bpf/progs/callx_rodata.c create mode 100644 tools/testing/selftests/bpf/progs/verifier_callx_rodata.c diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 580d1d82c186..3d92cdca62ed 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -33,7 +33,7 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ core_kern.c core_kern_overflow.c test_ringbuf.c \ test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ - test_ringbuf_overwrite.c + test_ringbuf_overwrite.c callx_rodata.c LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c new file mode 100644 index 000000000000..b58ebee9b1ef --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The kernel replaces the offsets of functions in a frozen read-only map with + * their addresses when the program is loaded. The program has to be the only + * user of the map, and no other program can use the map after that. + */ +#include +#include +#include + +#if defined(__x86_64__) || defined(__aarch64__) + +#define CALLEE_INSN 6 +#define DATA 0x1234 + +/* + * main: r2 = &value; r2 = *(u64 *)(r2 + 8); r1 = 10; callx r2; exit + * add1: r0 = r1; r0 += 1; exit + * + * where value is { DATA, offset of add1 in the program }. + */ +static const struct bpf_insn callx_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_2, 8), + BPF_MOV64_IMM(BPF_REG_1, 10), + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), +}; + +/* reads the data of the map */ +static const struct bpf_insn reader_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0), + BPF_EXIT_INSN(), +}; + +/* refers to the map and is rejected: r0 is not set */ +static const struct bpf_insn bad_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_EXIT_INSN(), +}; + +static char log_buf[16 * 1024]; +static struct bpf_func_info func_info[2]; +static int btf_fd; + +static int create_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + __u64 value[2] = { DATA, CALLEE_INSN * sizeof(struct bpf_insn) }; + int fd, zero = 0; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "callx_rodata", sizeof(int), sizeof(value), 1, + &opts); + if (!ASSERT_OK_FD(fd, "map_create")) + return -1; + if (!ASSERT_OK(bpf_map_update_elem(fd, &zero, value, 0), "map_update") || + !ASSERT_OK(bpf_map_freeze(fd), "map_freeze")) { + close(fd); + return -1; + } + return fd; +} + +static int load(const struct bpf_insn *prog_insns, int cnt, int map_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = sizeof(log_buf), + .log_level = 1, + ); + struct bpf_insn insns[ARRAY_SIZE(callx_insns)]; + + if (prog_insns == callx_insns) { + /* add1() is referred to by the data only and is found through func_info */ + opts.prog_btf_fd = btf_fd; + opts.func_info = func_info; + opts.func_info_cnt = 2; + opts.func_info_rec_size = sizeof(func_info[0]); + } + memcpy(insns, prog_insns, cnt * sizeof(insns[0])); + insns[0].imm = map_fd; + log_buf[0] = 0; + return bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "callx_map", "GPL", insns, cnt, &opts); +} + +#define LOAD(insns, map_fd) load(insns, ARRAY_SIZE(insns), map_fd) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +void test_callx_func_ptr_map(void) +{ + int int_id, proto_id, map_fd = -1, prog_fd = -1, reader_fd = -1, fd, zero = 0; + int main_id, add1_id; + struct btf *btf; + __u64 value[2]; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + proto_id = btf__add_func_proto(btf, int_id); + main_id = btf__add_func(btf, "main_prog", BTF_FUNC_GLOBAL, proto_id); + add1_id = btf__add_func(btf, "add1", BTF_FUNC_STATIC, proto_id); + if (!ASSERT_GT(main_id, 0, "btf_add_main") || + !ASSERT_GT(add1_id, 0, "btf_add_add1") || + !ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + func_info[0].insn_off = 0; + func_info[0].type_id = main_id; + func_info[1].insn_off = CALLEE_INSN; + func_info[1].type_id = add1_id; + btf_fd = btf__fd(btf); + + /* + * Another program relies on what the map has: it's verified with + * the data folded into constants. Pointers to functions are not looked + * for in such map. + */ + map_fd = create_map(); + if (map_fd < 0) + goto out; + reader_fd = LOAD(reader_insns, map_fd); + if (!ASSERT_OK_FD(reader_fd, "load_reader")) + goto out; + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_shared")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "unreachable insn 6", "log_shared"); + run(reader_fd, DATA, "run_reader"); + close(reader_fd); + reader_fd = -1; + close(map_fd); + + map_fd = create_map(); + if (map_fd < 0) + goto out; + + /* a program that is rejected is not a user, libbpf loads it again to get the log */ + fd = LOAD(bad_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_bad")) + close(fd); + + prog_fd = LOAD(callx_insns, map_fd); + if (!ASSERT_OK_FD(prog_fd, "load_callx")) { + printf("%s\n", log_buf); + goto out; + } + run(prog_fd, 11, "run_callx"); + + /* the offset of the function is gone from the map, the data is intact */ + if (ASSERT_OK(bpf_map_lookup_elem(map_fd, &zero, value), "map_lookup")) { + ASSERT_EQ(value[0], DATA, "data"); + ASSERT_NEQ(value[1], CALLEE_INSN * sizeof(struct bpf_insn), "pointer"); + } + + /* no other program can use the map now, another instance of the same one too */ + fd = LOAD(reader_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_reader_after")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "has addresses of functions of another program", "log_reader"); + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_second_instance")) + close(fd); + + run(prog_fd, 11, "run_callx_again"); +out: + if (reader_fd >= 0) + close(reader_fd); + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +#else + +void test_callx_func_ptr_map(void) +{ + test__skip(); +} + +#endif diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c new file mode 100644 index 000000000000..96f6179f0492 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include "callx_rodata.lskel.h" + +#if defined(__x86_64__) || defined(__aarch64__) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +/* + * Every program that has callx gets its own copy of .rodata with the offsets + * of its functions. The copies have what user space puts into .rodata before + * the load. The program that doesn't have callx uses .rodata. + */ +void test_callx_rodata_lskel(void) +{ + struct callx_rodata_lskel *skel; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + skel = callx_rodata_lskel__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + skel->rodata->bias = 7; + + if (!ASSERT_OK(callx_rodata_lskel__load(skel), "load")) + goto out; + + skel->bss->op_idx = 0; + run(skel->progs.select_op.prog_fd, 17, "add_bias"); + skel->bss->op_idx = 1; + run(skel->progs.select_op.prog_fd, 30, "mul3"); + skel->bss->op_idx = 2; + run(skel->progs.select_op.prog_fd, -1, "out_of_range"); + /* mul3(add_bias(4)) */ + run(skel->progs.both_ops.prog_fd, 33, "both_ops"); + run(skel->progs.read_bias.prog_fd, 7, "read_bias"); +out: + callx_rodata_lskel__destroy(skel); +} + +#else + +void test_callx_rodata_lskel(void) +{ + test__skip(); +} + +#endif diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index bc96a752b1ae..9c3ff7d83fbe 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -29,6 +29,7 @@ #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" #include "verifier_callx.skel.h" +#include "verifier_callx_rodata.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -209,6 +210,7 @@ void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } void test_verifier_callx(void) { RUN_JITED(verifier_callx); } +void test_verifier_callx_rodata(void) { RUN_JITED(verifier_callx_rodata); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } diff --git a/tools/testing/selftests/bpf/progs/callx_rodata.c b/tools/testing/selftests/bpf/progs/callx_rodata.c new file mode 100644 index 000000000000..07ae2d1b6b8e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/callx_rodata.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* callx through pointers to functions in .rodata, loaded by light skeleton */ + +#include +#include + +typedef int (*op_fn)(int); + +/* set by user space before the programs are loaded, it's in .rodata too */ +const volatile int bias = 1; + +int op_idx; + +static __noinline int add_bias(int x) +{ + return x + bias; +} + +static __noinline int mul3(int x) +{ + return x * 3; +} + +/* volatile, so that the compiler doesn't replace the table with direct calls */ +static op_fn const volatile ops[] = { add_bias, mul3 }; + +SEC("socket") +int select_op(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + return ops[i](10); +} + +/* functions have other offsets in this program */ +SEC("socket") +int both_ops(void *ctx) +{ + return ops[1](ops[0](4)); +} + +/* no callx here */ +SEC("socket") +int read_bias(void *ctx) +{ + return bias; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c new file mode 100644 index 000000000000..af1f9305da37 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -0,0 +1,846 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx through pointers to functions in read-only data */ + +#include +#include +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) + +/* + * Read-only data with pointers to functions, where the compiler puts tables + * of functions, structures of operations and vtables. libbpf resolves + * a pointer to the offset of the function in the program and the kernel + * recognizes it by that value. + */ +#define DATA(SECTION, NAME, ...) \ + ".pushsection " SECTION ",@progbits;" \ + ".balign 8;" \ + #NAME "_%=:" \ + __VA_ARGS__ \ + ".type " #NAME "_%=, @object;" \ + ".size " #NAME "_%=, .-" #NAME "_%=;" \ + ".popsection;" + +#define RODATA(NAME, ...) DATA(".rodata,\"a\"", NAME, __VA_ARGS__) + +#define FUNC_TABLE2(NAME, F0, F1) RODATA(NAME, ".quad " #F0 "; .quad " #F1 ";") + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* the second element of the table is called */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_const_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* + * The program reads the address of a function from the data, so pointers to + * functions in data are recognized only for programs that may leak pointers. + * Otherwise nothing refers to the functions. + */ +SEC("socket") +__success __retval(12) +__failure_unpriv __msg_unpriv("unreachable insn") +__caps_unpriv(CAP_BPF) +__naked void callx_rodata_needs_perfmon(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* the address of an element is used instead of the address of the table */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_elem_addr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= + 8 ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* pointers to functions are mixed with other data like in a vtable */ +SEC("socket") +__success __retval(23) +__log_level(2) +__msg("r1 = *(u64 *)(r6 +0) ; R1=7") +__msg("r2 = *(u64 *)(r6 +8) ; R2=func()") +__naked void callx_rodata_mixed_with_data(void) +{ + asm volatile ( + RODATA(vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(7) */ + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + /* add2(13) */ + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* + * Position independent code keeps constants with pointers in .data.rel.ro. + * libbpf treats it as read-only data. + */ +SEC("socket") +__success __retval(23) +__naked void callx_data_rel_ro(void) +{ + asm volatile ( + DATA(".data.rel.ro,\"aw\"", vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* an array of structures: the index selects one of the functions, not the data */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_array_of_structs(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 0x1111; .quad add2; .quad 0x2222;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r2 = arr_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* and the data of the element that the index selects can be read */ +SEC("socket") +__success __retval(16) +__naked void callx_rodata_array_of_structs_data(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r7 = arr_%= ll;" + "r7 += r3;" + "r2 = *(u64 *)(r7 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "r1 = *(u64 *)(r7 + 8);" + "r0 += r1;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* but not the data and the pointer with the same load */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_array_of_structs_any(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = arr_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* dynamic dispatch: which vtable is used is not known until run time */ +SEC("socket") +__success __retval(2) +__naked void callx_rodata_two_vtables(void) +{ + asm volatile ( + RODATA(vt_a, ".quad 1; .quad add1;") + RODATA(vt_b, ".quad 0; .quad add2;") + "call %[bpf_get_prandom_u32];" + "r6 = vt_a_%= ll;" + "r0 &= 1;" + "if r0 == 0 goto +2;" + "r6 = vt_b_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(1) or add2(0) */ + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* both functions are verified, either of them is called */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_var_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a table that has no symbol, the compiler generates such for a switch statement */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_no_symbol(void) +{ + asm volatile ( + ".pushsection .rodata,\"a\",@progbits;" + ".balign 8;" + ".Lanon_%=:" + ".quad add1;" + ".quad add2;" + ".popsection;" + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = .Lanon_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the first instruction of the function is removed by the verifier */ +__naked __noinline __used +static unsigned long nop_add1(void) +{ + asm volatile ( + "goto +0;" + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +/* the pointer follows the function when instructions are removed */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_func_starts_with_nop(void) +{ + asm volatile ( + RODATA(tbl, ".quad nop_add1; .quad 0;") + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* can't be called with a scalar in r1 */ +__naked __noinline __used +static unsigned long deref_r1(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long ret0(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +/* every function that might be called is verified */ +SEC("socket") +__failure __msg("R1 invalid mem access 'scalar'") +__naked void callx_rodata_all_callees_verified(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a function that is never called is not verified, it's dead code */ +SEC("socket") +__success __retval(0) +__naked void callx_rodata_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* a pointer that is read and never called doesn't keep the function, it's NULL */ +SEC("socket") +__success __retval(1) +__naked void callx_rodata_read_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 0;" + "callx r2;" + "r2 = *(u64 *)(r6 + 8);" + "if r2 != 0 goto +1;" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* the index may select what is not a pointer to a function */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_index_beyond_table(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0x1234; .quad 0x5678;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the address of a function is not known until the program is jitted */ +SEC("socket") +__failure __msg("read of 4 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_partial_read(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 10;" + "callx r2;" + "r0 = *(u32 *)(r6 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("read of 8 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_misaligned_read(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 4);" + "exit;" + ::: __clobber_all); +} + +/* + * The kernel doesn't look for pointers to functions in the data of a program + * that doesn't have callx. It's a number there. + */ +SEC("socket") +__success +__naked void callx_rodata_no_callx(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r0 = *(u32 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* the data next to a pointer is still known to the verifier */ +SEC("socket") +__success __retval(0x1234) +__log_level(2) +__msg("R0=4660") +__naked void callx_rodata_data_is_const(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad 0x1234;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The map has the offset of the function until the program is jitted. + * bpf_compute_const_regs() must not take it for a constant, otherwise + * bpf_prune_dead_branches() turns the branch below into the wrong one. + */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_ptr_is_not_const(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r0 = 0;" + /* the offset is less, the address is greater */ + "if r2 < 0x100000 goto 1f;" + "r1 = 10;" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +/* the pointer read from the data can't be modified */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_rodata_modified_ptr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r2 += 8;" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* + * A pointer to a global function is not recognized, it's a number. The function + * is called directly as well, so that libbpf adds it to the program and stores + * its offset into the table. + */ +SEC("socket") +__failure __msg("R2 has type scalar, expected func") +__naked void callx_rodata_global_func(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, global_add3) + "r1 = 1;" + "call global_add3;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* selfcall(n) { return n ? tbl[0](n - 1) : 0; }, where tbl[0] == selfcall */ +__naked __noinline __used +static unsigned long selfcall(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, selfcall, ret0) + "r0 = 0;" + "if r1 == 0 goto 1f;" + "r1 += -1;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("recursive call from selfcall() to selfcall()") +__naked void callx_rodata_recursion(void) +{ + asm volatile ( + "r1 = 2;" + "call selfcall;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + +/* stack of all possible callees is accounted */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_rodata_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, use_stack_304) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* stack of a callback that is read from the data is accounted too */ +SEC("socket") +__failure __msg("combined stack size of 2 calls is") +__naked void callx_rodata_callback_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, use_stack_304, ret0) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r6 = tbl_%= ll;" + "r1 = 1;" + "r2 = *(u64 *)(r6 + 0);" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + +static __noinline __used int pull_data(struct __sk_buff *skb) +{ + return bpf_skb_pull_data(skb, 0); +} + +static __noinline __used int calls_pull_data(struct __sk_buff *skb) +{ + return pull_data(skb) + 1; +} + +__noinline __used int global_calls_pull_data(struct __sk_buff *skb) +{ + return calls_pull_data(skb) + 2; +} + +/* + * check_cfg() explores the functions that the data points to when it sees + * the address of the data. It must not lose the effects of the functions + * that call each other: global_calls_pull_data() changes packet data. + */ +SEC("tc") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void callx_rodata_callee_effects(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, pull_data, calls_pull_data) + "r6 = r1;" + "r7 = *(u32 *)(r6 + %[data]);" + "r8 = *(u32 *)(r6 + %[data_end]);" + "r2 = r7;" + "r2 += 8;" + "if r2 > r8 goto 1f;" + "r1 = r6;" + "call global_calls_pull_data;" + "r0 = *(u8 *)(r7 + 0);" + "1:" + "r9 = tbl_%= ll;" + "r2 = *(u64 *)(r9 + 8);" + "r1 = r6;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* pointers to functions in C */ + +typedef int (*op_fn)(int); + +#define DEFINE_OP(N) static __noinline int op##N(int x) { return x * (N + 2) + N; } + +DEFINE_OP(0) DEFINE_OP(1) DEFINE_OP(2) DEFINE_OP(3) +DEFINE_OP(4) DEFINE_OP(5) DEFINE_OP(6) DEFINE_OP(7) +DEFINE_OP(8) DEFINE_OP(9) DEFINE_OP(10) DEFINE_OP(11) +DEFINE_OP(12) DEFINE_OP(13) DEFINE_OP(14) DEFINE_OP(15) + +static op_fn const ops[] = { + op0, op1, op2, op3, op4, op5, op6, op7, + op8, op9, op10, op11, op12, op13, op14, op15, +}; + +int op_idx = 11; + +SEC("socket") +__success __retval(50) +int callx_c_table(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* op11(3) = 3 * 13 + 11 */ + return ops[i](3); +} + +SEC("socket") +__success __retval(50) +int callx_c_table_null_check(void *ctx) +{ + unsigned int i = op_idx; + op_fn op; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* the address of a function is not known until the program is jitted */ + op = ops[i]; + if (!op) + return -2; + return op(3); +} + +/* a structure of operations, where pointers to functions are mixed with data */ +struct shape_ops { + int id; + op_fn area; + long scale; + op_fn perimeter; +}; + +static const struct shape_ops square_ops = { 1, op1, 10, op2 }; +static const struct shape_ops circle_ops = { 2, op3, 20, op1 }; + +static __noinline int use_shape(const struct shape_ops *ops, int x) +{ + return ops->area(x) * ops->scale + ops->perimeter(ops->id); +} + +SEC("socket") +__success __retval(433) +int callx_c_ops_mixed_with_data(void *ctx) +{ + /* + * square: op1(2) * 10 + op2(1) = 7 * 10 + 6 = 76 + * circle: op3(3) * 20 + op1(2) = 18 * 20 + 7 = 367 + * minus 10, since op_idx is 11 + */ + return use_shape(&square_ops, 2) + use_shape(&circle_ops, 3) - (op_idx == 11 ? 10 : 0); +} + +/* the ops are selected at run time */ +SEC("socket") +__success __retval(367) +int callx_c_ops_selected(void *ctx) +{ + const struct shape_ops *ops = op_idx == 11 ? &circle_ops : &square_ops; + + return use_shape(ops, 3); +} + +/* the compiler might turn the switch into a table that has no symbol */ +static __noinline int call_by_switch(unsigned int idx, int x) +{ + op_fn op; + + switch (idx) { + case 0: + op = op8; + break; + case 1: + op = op9; + break; + case 2: + op = op10; + break; + case 3: + op = op11; + break; + case 4: + op = op12; + break; + case 5: + op = op13; + break; + case 6: + op = op14; + break; + case 7: + op = op15; + break; + default: + return -1; + } + return op(x); +} + +SEC("socket") +__success __retval(50) +int callx_c_switch_table(void *ctx) +{ + /* op11(3) = 3 * 13 + 11 */ + return call_by_switch(op_idx - 8, 3); +} + +/* + * Misaligned pointers to functions are ignored, the rest of the data is + * accessible as before. The program doesn't have callx, so libbpf doesn't + * append the functions that .rodata points to and doesn't make a copy of + * .rodata for it. + */ +static const struct { + char tag; + op_fn op; +} __attribute__((packed)) packed_ops[] = { + { 5, op0 }, + { 7, op1 }, +}; + +SEC("socket") +__success __retval(7) +int callx_c_packed_struct(void *ctx) +{ + return packed_ops[op_idx & 1].tag; +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; From fd318eab4b0905ffeb2fe1e93ad0d2e217f1f584 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 03:10:42 +0000 Subject: [PATCH 245/660] bpf, docs: Document callx instruction Document callx: what it does, where the callee address comes from and what is not supported. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924031042.1690890-18-alexei.starovoitov@gmail.com --- Documentation/bpf/clang-notes.rst | 7 +++++-- Documentation/bpf/linux-notes.rst | 31 +++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst index 2c872a1ee08e..3ccc7b09d19e 100644 --- a/Documentation/bpf/clang-notes.rst +++ b/Documentation/bpf/clang-notes.rst @@ -23,8 +23,11 @@ For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support Jump instructions ================= -If ``-O0`` is used, Clang will generate the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) -instruction, which is not supported by the Linux kernel verifier. +Clang generates the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) instruction for calls +through a function pointer. The Linux kernel verifier accepts it only when it +can prove that the register holds the address of a static BPF function, see +Documentation/bpf/linux-notes.rst. If ``-O0`` is used, Clang will generate this +instruction for helper calls as well, which is not supported. Atomic operations ================= diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst index 00d2693de025..6c036b54a29f 100644 --- a/Documentation/bpf/linux-notes.rst +++ b/Documentation/bpf/linux-notes.rst @@ -15,10 +15,33 @@ Byte swap instructions Jump instructions ================= -``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), where the helper function -integer would be read from a specified register, is not currently supported -by the verifier. Any programs with this instruction will fail to load -until such support is added. +``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), ``callx dst``, performs an indirect +call of a BPF function whose address is held in the ``dst`` register. The +``src``, ``offset`` and ``imm`` fields are reserved and must be zero. + +The address of a BPF function gets into a register in one of two ways: + +* it is loaded by a 64-bit immediate instruction with ``src`` = + ``BPF_PSEUDO_FUNC``; +* it is read, with a 64-bit load, from a frozen read-only array map, that no + other program uses, that holds its read-only data: tables of functions, + structures of operations, vtables, where pointers to functions may be mixed + with other data. In the map a pointer to a function is the offset in bytes + of its first instruction in the program, and that is how the verifier + recognizes it. It is replaced with the address of the function when + the program is loaded. The program reads it from there, which requires + ``CAP_PERFMON``. + +In both cases only static functions can be referenced. Therefore all functions +that can be called indirectly are known to the verifier before it starts to +analyze the program, and ``callx`` is verified as a direct call of every +function that ``dst`` may point to at that instruction. The same rules apply: +the calls can not be recursive, and the depth of the call chain and its +combined stack size are limited. + +Calling helper or kernel functions through a register, indirect calls of global +functions, and tail calls in functions that are called via ``callx`` are not +supported. ``callx`` requires the BPF JIT. Maps ==== From c43c348a84460b003fa757dd43d8c89848ae220d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Thu, 24 Sep 2026 10:37:36 +0800 Subject: [PATCH 246/660] bpf: Keep target extended until its last freplace link detaches The is_extended / prog_array_member_cnt protocol introduced by commit d6083f040d5d ("bpf: Prevent tailcall infinite loop caused by freplace") keeps a prog extended by a freplace program out of prog_array maps, and vice versa: once a tail call re-enters an extended subprogram, its tail_call_cnt resets on every execution and the loop never terminates. But is_extended is a plain boolean, while one target prog can carry several freplace links at the same time, one on its entry and one on a global subprogram. __bpf_trampoline_unlink_prog() cleared is_extended whenever *any* freplace link detached, so detaching one of two links re-armed the unbounded loop through the remaining one. Replace the is_extended boolean with a count of the freplace links attached to each target prog, so the target stays extended until its last link detaches. Also rename bpf_freplace_check_tgt_prog() to bpf_freplace_link_tgt_prog(), as the helper has never been a pure check: it reserves the target prog on success. Fixes: d6083f040d5d ("bpf: Prevent tailcall infinite loop caused by freplace") Signed-off-by: Yuan Chen Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260924023737.1140521-2-chenyuan_fl@163.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 4 ++-- kernel/bpf/arraymap.c | 2 +- kernel/bpf/trampoline.c | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 9525d3ecd923..1e1ce2afe2ed 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1833,7 +1833,6 @@ struct bpf_prog_aux { bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; @@ -1845,7 +1844,7 @@ struct bpf_prog_aux { u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1877,6 +1876,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0ce26b538075..0fe9afd4a591 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, } mutex_lock(&prog->aux->ext_mutex); - is_extended = prog->aux->is_extended; + is_extended = prog->aux->freplace_link_cnt > 0; if (!is_extended) prog->aux->prog_array_member_cnt++; mutex_unlock(&prog->aux->ext_mutex); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..da85bd580ef0 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -843,7 +843,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -857,7 +857,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -970,7 +970,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; tr->extension_prog = node->link->prog; @@ -1016,7 +1016,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); From 01d37848bf5b4827a66f24f9654919936e12fa8e Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Thu, 24 Sep 2026 10:37:37 +0800 Subject: [PATCH 247/660] selftests/bpf: Verify is_extended with multiple freplace links Extend tc_bpf2bpf with two freplace links, one on entry_tc and one on subprog_tc, and detach the one on the entry: while subprog_tc is still extended, updating entry_tc into a prog_array map must keep failing with -EBUSY, and succeed again once the last link detaches. The test asserts the rejection instead of running the prog, as the update succeeds and the prog loops unbounded on an unfixed kernel. Signed-off-by: Yuan Chen Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260924023737.1140521-3-chenyuan_fl@163.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/tailcalls.c | 74 +++++++++++++++++++ .../bpf/progs/tailcall_freplace_multi.c | 27 +++++++ 2 files changed, 101 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index c5c9d6c359bb..aefb46778307 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -6,6 +6,7 @@ #include "tailcall_bpf2bpf_hierarchy2.skel.h" #include "tailcall_bpf2bpf_hierarchy3.skel.h" #include "tailcall_freplace.skel.h" +#include "tailcall_freplace_multi.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" #include "tailcall_cgrp_storage_owner.skel.h" @@ -1655,6 +1656,77 @@ out: tc_bpf2bpf__destroy(tc_skel); } +static void test_tailcall_freplace_multi(void) +{ + struct tailcall_freplace_multi *freplace_skel = NULL; + struct bpf_link *link_subprog = NULL, *link_entry = NULL; + struct tc_bpf2bpf *tc_skel = NULL; + int tc_prog_fd, map_fd, key = 0, err; + + tc_skel = tc_bpf2bpf__open_and_load(); + if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) + return; + + tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); + + freplace_skel = tailcall_freplace_multi__open(); + if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace_multi__open")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK(err, "set_attach_target subprog_tc")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK(err, "set_attach_target entry_tc")) + goto out; + + err = tailcall_freplace_multi__load(freplace_skel); + if (!ASSERT_OK(err, "tailcall_freplace_multi__load")) + goto out; + + map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); + + link_subprog = bpf_program__attach_freplace(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK_PTR(link_subprog, "attach_freplace subprog_tc")) + goto out; + + link_entry = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK_PTR(link_entry, "attach_freplace entry_tc")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with extended prog")) + goto out; + + err = bpf_link__destroy(link_entry); + link_entry = NULL; + if (!ASSERT_OK(err, "destroy entry link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with still extended prog")) + goto out; + + err = bpf_link__destroy(link_subprog); + link_subprog = NULL; + if (!ASSERT_OK(err, "destroy subprog link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + ASSERT_OK(err, "update jmp_table"); + +out: + bpf_link__destroy(link_subprog); + bpf_link__destroy(link_entry); + tailcall_freplace_multi__destroy(freplace_skel); + tc_bpf2bpf__destroy(tc_skel); +} + static void test_tailcall_failure() { RUN_TESTS(tailcall_fail); @@ -2005,6 +2077,8 @@ void test_tailcalls(void) test_tailcall_freplace(); if (test__start_subtest("tailcall_bpf2bpf_freplace")) test_tailcall_bpf2bpf_freplace(); + if (test__start_subtest("tailcall_freplace_multi")) + test_tailcall_freplace_multi(); if (test__start_subtest("tailcall_failure")) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) diff --git a/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c new file mode 100644 index 000000000000..ca764411dc0b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +SEC("freplace") +int subprog_freplace(struct __sk_buff *skb) +{ + bpf_tail_call_static(skb, &jmp_table, 0); + return 0; +} + +SEC("freplace") +int entry_freplace(struct __sk_buff *skb) +{ + return 0; +} + +char __license[] SEC("license") = "GPL"; From dbf8cef936c7783ca909c38097a69f73173a19a7 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 23 Sep 2026 09:42:42 +0900 Subject: [PATCH 248/660] bpftool: Fix CPU IDs in per-CPU map output bpftool uses dense per-CPU value-buffer slots when printing per-CPU map values. It also uses the slot index as the CPU ID, which produces incorrect labels when the possible CPU mask is sparse, such as 0,2-3. Parse the possible CPU mask and use the corresponding logical CPU ID in plain, JSON, and BTF-formatted output. Keep the dense slot index for accessing the per-CPU value buffer. Pass the parsed CPU count together with the CPU ID array so that the printed CPU IDs and loop bounds come from the same possible-CPU mask. Propagate CPU-ID lookup and map output errors through the shared output path to its callers. Fixes: 71bb428fe2c1 ("tools: bpf: add bpftool") Signed-off-by: Hui Su Link: https://lore.kernel.org/bpf/20260813155131.1022745-3-sh_def@163.com/ Link: https://lore.kernel.org/bpf/20260923004243.969919-2-sh_def@163.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/bpftool/common.c | 40 ++++++++++++++++ tools/bpf/bpftool/main.h | 1 + tools/bpf/bpftool/map.c | 95 +++++++++++++++++++++++++------------- 3 files changed, 103 insertions(+), 33 deletions(-) diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index ef366ccc9650..f9b87efa7c9a 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -31,6 +31,7 @@ #include #include #include /* libbpf_num_possible_cpus */ +#include #include #include @@ -655,6 +656,45 @@ unsigned int get_possible_cpus(void) return cpus; } +int get_possible_cpu_ids(int **cpu_ids) +{ + const char *possible_cpus_file = "/sys/devices/system/cpu/possible"; + bool *mask = NULL; + int mask_sz, nr_cpus = 0; + int *ids = NULL; + int i, res; + + *cpu_ids = NULL; + + res = parse_cpu_mask_file(possible_cpus_file, &mask, &mask_sz); + if (res) { + p_err("failed to parse possible CPU mask: %s", strerror(-res)); + return res; + } + + for (i = 0; i < mask_sz; i++) + nr_cpus += mask[i]; + + ids = calloc(nr_cpus, sizeof(*ids)); + if (!ids) { + p_err("mem alloc failed"); + res = -ENOMEM; + goto out; + } + + for (i = 0, nr_cpus = 0; i < mask_sz; i++) { + if (mask[i]) + ids[nr_cpus++] = i; + } + *cpu_ids = ids; + ids = NULL; + res = nr_cpus; +out: + free(ids); + free(mask); + return res; +} + static char * ifindex_to_name_ns(__u32 ifindex, __u32 ns_dev, __u32 ns_ino, char *buf) { diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 9315a1db1f7c..48eedcb9f6c0 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -216,6 +216,7 @@ void print_hex_data_json(uint8_t *data, size_t len); unsigned int get_page_size(void); unsigned int get_possible_cpus(void); +int get_possible_cpu_ids(int **cpu_ids); const char * ifindex_to_arch(__u32 ifindex, __u64 ns_dev, __u64 ns_ino, const char **opt); diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 684a8fb72414..20d59eab09a1 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -70,7 +70,7 @@ static void *alloc_value(struct bpf_map_info *info) static int do_dump_btf(const struct btf_dumper *d, struct bpf_map_info *map_info, void *key, - void *value) + void *value, const int *cpu_ids, int cpu_cnt) { __u32 value_id; int ret = 0; @@ -93,15 +93,15 @@ static int do_dump_btf(const struct btf_dumper *d, jsonw_name(d->jw, "value"); ret = btf_dumper_type(d, value_id, value); } else { - unsigned int i, n, step; + unsigned int step; + int i; jsonw_name(d->jw, "values"); jsonw_start_array(d->jw); - n = get_possible_cpus(); step = round_up(map_info->value_size, 8); - for (i = 0; i < n; i++) { + for (i = 0; i < cpu_cnt; i++) { jsonw_start_object(d->jw); - jsonw_int_field(d->jw, "cpu", i); + jsonw_int_field(d->jw, "cpu", cpu_ids[i]); jsonw_name(d->jw, "value"); ret = btf_dumper_type(d, value_id, value + i * step); jsonw_end_object(d->jw); @@ -130,7 +130,8 @@ static json_writer_t *get_btf_writer(void) } static void print_entry_json(struct bpf_map_info *info, unsigned char *key, - unsigned char *value, struct btf *btf) + unsigned char *value, struct btf *btf, + const int *cpu_ids, int cpu_cnt) { jsonw_start_object(json_wtr); @@ -150,12 +151,12 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, }; jsonw_name(json_wtr, "formatted"); - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); } } else { - unsigned int i, n, step; + unsigned int step; + int i; - n = get_possible_cpus(); step = round_up(info->value_size, 8); jsonw_name(json_wtr, "key"); @@ -163,10 +164,10 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, jsonw_name(json_wtr, "values"); jsonw_start_array(json_wtr); - for (i = 0; i < n; i++) { + for (i = 0; i < cpu_cnt; i++) { jsonw_start_object(json_wtr); - jsonw_int_field(json_wtr, "cpu", i); + jsonw_int_field(json_wtr, "cpu", cpu_ids[i]); jsonw_name(json_wtr, "value"); print_hex_data_json(value + i * step, @@ -183,7 +184,7 @@ static void print_entry_json(struct bpf_map_info *info, unsigned char *key, }; jsonw_name(json_wtr, "formatted"); - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); } } @@ -245,7 +246,7 @@ print_entry_error(struct bpf_map_info *map_info, void *key, int lookup_errno) } static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, - unsigned char *value) + unsigned char *value, const int *cpu_ids, int cpu_cnt) { if (!map_is_per_cpu(info->type)) { bool single_line, break_names; @@ -273,9 +274,9 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, printf("\n"); } else { - unsigned int i, n, step; + unsigned int step; + int i; - n = get_possible_cpus(); step = round_up(info->value_size, 8); if (info->key_size) { @@ -284,9 +285,9 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, printf("\n"); } if (info->value_size) { - for (i = 0; i < n; i++) { - printf("value (CPU %02u):%c", - i, info->value_size > 16 ? '\n' : ' '); + for (i = 0; i < cpu_cnt; i++) { + printf("value (CPU %02d):%c", + cpu_ids[i], info->value_size > 16 ? '\n' : ' '); fprint_hex(stdout, value + i * step, info->value_size, " "); printf("\n"); @@ -742,7 +743,7 @@ static int do_show(int argc, char **argv) static int dump_map_elem(int fd, void *key, void *value, struct bpf_map_info *map_info, struct btf *btf, - json_writer_t *btf_wtr) + json_writer_t *btf_wtr, const int *cpu_ids, int cpu_cnt) { if (bpf_map_lookup_elem(fd, key, value)) { print_entry_error(map_info, key, errno); @@ -750,7 +751,7 @@ static int dump_map_elem(int fd, void *key, void *value, } if (json_output) { - print_entry_json(map_info, key, value, btf); + print_entry_json(map_info, key, value, btf, cpu_ids, cpu_cnt); } else if (btf) { struct btf_dumper d = { .btf = btf, @@ -758,9 +759,9 @@ static int dump_map_elem(int fd, void *key, void *value, .is_plain_text = true, }; - do_dump_btf(&d, map_info, key, value); + do_dump_btf(&d, map_info, key, value, cpu_ids, cpu_cnt); } else { - print_entry_plain(map_info, key, value); + print_entry_plain(map_info, key, value, cpu_ids, cpu_cnt); } return 0; @@ -833,6 +834,8 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, void *key, *value, *prev_key; unsigned int num_elems = 0; struct btf *btf = NULL; + int *cpu_ids = NULL; + int cpu_cnt = 0; int err; key = malloc(info->key_size); @@ -845,6 +848,14 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, prev_key = NULL; + if (map_is_per_cpu(info->type)) { + cpu_cnt = get_possible_cpu_ids(&cpu_ids); + if (cpu_cnt < 0) { + err = cpu_cnt; + goto exit_free; + } + } + if (wtr) { err = get_map_kv_btf(info, &btf); if (err) { @@ -876,7 +887,8 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, err = 0; break; } - if (!dump_map_elem(fd, key, value, info, btf, wtr)) + if (!dump_map_elem(fd, key, value, info, btf, wtr, + cpu_ids, cpu_cnt)) num_elems++; prev_key = key; } @@ -893,6 +905,7 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); + free(cpu_ids); free_map_kv_btf(btf); return err; @@ -1035,17 +1048,30 @@ exit_free: return err; } -static void print_key_value(struct bpf_map_info *info, void *key, - void *value) +static int print_key_value(struct bpf_map_info *info, void *key, + void *value) { json_writer_t *btf_wtr; struct btf *btf; + int *cpu_ids = NULL; + int cpu_cnt = 0; + int err = 0; - if (get_map_kv_btf(info, &btf)) - return; + if (map_is_per_cpu(info->type)) { + cpu_cnt = get_possible_cpu_ids(&cpu_ids); + + if (cpu_cnt < 0) { + err = cpu_cnt; + goto out; + } + } + + err = get_map_kv_btf(info, &btf); + if (err) + goto out; if (json_output) { - print_entry_json(info, key, value, btf); + print_entry_json(info, key, value, btf, cpu_ids, cpu_cnt); } else if (btf) { /* if here json_wtr wouldn't have been initialised, * so let's create separate writer for btf @@ -1055,7 +1081,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, p_info("failed to create json writer for btf. falling back to plain output"); free_map_kv_btf(btf); btf = NULL; - print_entry_plain(info, key, value); + print_entry_plain(info, key, value, cpu_ids, cpu_cnt); } else { struct btf_dumper d = { .btf = btf, @@ -1063,13 +1089,16 @@ static void print_key_value(struct bpf_map_info *info, void *key, .is_plain_text = true, }; - do_dump_btf(&d, info, key, value); + do_dump_btf(&d, info, key, value, cpu_ids, cpu_cnt); jsonw_destroy(&btf_wtr); } } else { - print_entry_plain(info, key, value); + print_entry_plain(info, key, value, cpu_ids, cpu_cnt); } free_map_kv_btf(btf); +out: + free(cpu_ids); + return err; } static int do_lookup(int argc, char **argv) @@ -1114,7 +1143,7 @@ static int do_lookup(int argc, char **argv) } /* here means bpf_map_lookup_elem() succeeded */ - print_key_value(&info, key, value); + err = print_key_value(&info, key, value); exit_free: free(key); @@ -1404,7 +1433,7 @@ static int do_pop_dequeue(int argc, char **argv) goto exit_free; } - print_key_value(&info, key, value); + err = print_key_value(&info, key, value); exit_free: free(key); From edc071d7b3aaae8047a12c21b02cc537557aacee Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 23 Sep 2026 09:42:43 +0900 Subject: [PATCH 249/660] bpftool: Fix sparse CPU IDs in prog profile bpftool prog profile currently treats the number of possible CPUs as both the logical CPU ID range and the stride of the perf event array. That misses valid logical CPUs when the possible CPU mask is sparse, such as 0,2-3, and can use incorrect PERF_EVENT_ARRAY keys. Keep the compact possible CPU count for per-CPU result buffers, while enumerating the actual logical CPU IDs when creating per-CPU perf event groups. Use the maximum logical CPU ID plus one as the metric stride in the PERF_EVENT_ARRAY. This keeps the current per-CPU event grouping intact while separating the compact per-CPU buffer index from the logical CPU ID and event-array key. Report the logical CPU ID when a per-CPU event was not counted. Fixes: 47c09d6a9f67 ("bpftool: Introduce "prog profile" command") Signed-off-by: Hui Su Link: https://lore.kernel.org/bpf/20260813160858.1042834-3-sh_def@163.com/ Link: https://lore.kernel.org/bpf/20260923004243.969919-3-sh_def@163.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/bpftool/prog.c | 51 +++++++++++++++-------- tools/bpf/bpftool/skeleton/profiler.bpf.c | 8 ++-- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 000774835b10..24e40dfab469 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -2164,6 +2164,9 @@ struct profile_metric { }; static __u64 profile_total_count; +static int profile_cpu_cnt; +static int *profile_cpu_ids; +static int profile_cpu_id_span; #define MAX_NUM_PROFILE_METRICS 4 @@ -2200,7 +2203,7 @@ static int profile_parse_metrics(int argc, char **argv) static int profile_read_values(struct profiler_bpf *obj) { - __u32 m, cpu, num_cpu = obj->rodata->num_cpu; + __u32 m, cpu, num_cpu = profile_cpu_cnt; int reading_map_fd, count_map_fd; __u64 counts[num_cpu]; __u32 key = 0; @@ -2235,8 +2238,8 @@ static int profile_read_values(struct profiler_bpf *obj) for (cpu = 0; cpu < num_cpu; cpu++) { val = &values[cpu]; if (counts[cpu] && !val->running) { - p_err("perf event %s was not counted on CPU %u", - metrics[m].name, cpu); + p_err("perf event %s was not counted on CPU %d", + metrics[m].name, profile_cpu_ids[cpu]); return -EAGAIN; } metrics[m].val.enabled += val->enabled; @@ -2394,6 +2397,7 @@ static void profile_close_perf_events(void) close(profile_perf_events[i]); free(profile_perf_events); + profile_perf_events = NULL; profile_perf_event_cnt = 0; } @@ -2427,13 +2431,12 @@ static int profile_open_perf_event(int mid, int cpu, static int profile_open_perf_events(struct profiler_bpf *obj) { - __u32 num_cpu = obj->rodata->num_cpu; __u32 map_key; unsigned int cpu, m; int group_fd, map_fd, pmu_fd; int err; - profile_perf_events = calloc(num_cpu * obj->rodata->num_metric, + profile_perf_events = calloc(profile_cpu_cnt * obj->rodata->num_metric, sizeof(*profile_perf_events)); if (!profile_perf_events) { p_err("failed to allocate memory for perf_event array: %s", @@ -2442,32 +2445,34 @@ static int profile_open_perf_events(struct profiler_bpf *obj) } map_fd = bpf_map__fd(obj->maps.events); - for (cpu = 0; cpu < num_cpu; cpu++) { + for (cpu = 0; cpu < (unsigned int)profile_cpu_cnt; cpu++) { + int cpu_id = profile_cpu_ids[cpu]; + group_fd = -1; - map_key = cpu; + map_key = cpu_id; for (m = 0; m < ARRAY_SIZE(metrics); m++) { if (!metrics[m].selected) continue; - pmu_fd = profile_open_perf_event(m, cpu, map_fd, group_fd, + pmu_fd = profile_open_perf_event(m, cpu_id, map_fd, group_fd, map_key); if (pmu_fd == -ENODEV && group_fd < 0) break; if (pmu_fd < 0) { - p_err("failed to add event %s to group on CPU %u: %s", - metrics[m].name, cpu, strerror(-pmu_fd)); + p_err("failed to add event %s to group on CPU %d: %s", + metrics[m].name, cpu_id, strerror(-pmu_fd)); return pmu_fd; } if (group_fd < 0) group_fd = pmu_fd; - map_key += num_cpu; + map_key += profile_cpu_id_span; } if (group_fd < 0) continue; if (ioctl(group_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP)) { err = -errno; - p_err("failed to enable perf event group on CPU %u: %s", - cpu, strerror(-err)); + p_err("failed to enable perf event group on CPU %d: %s", + cpu_id, strerror(-err)); return err; } } @@ -2486,6 +2491,10 @@ static int profile_print_and_cleanup(void) close(profile_tgt_fd); free(profile_tgt_name); + free(profile_cpu_ids); + profile_cpu_ids = NULL; + profile_cpu_cnt = 0; + profile_cpu_id_span = 0; return err; } @@ -2496,7 +2505,7 @@ static void int_exit(int signo) static int do_profile(int argc, char **argv) { - int num_metric, num_cpu, err = -1; + int num_metric, err = -1; struct bpf_program *prog; unsigned long duration; char *endptr; @@ -2527,11 +2536,12 @@ static int do_profile(int argc, char **argv) if (num_metric <= 0) goto out; - num_cpu = libbpf_num_possible_cpus(); - if (num_cpu <= 0) { + profile_cpu_cnt = get_possible_cpu_ids(&profile_cpu_ids); + if (profile_cpu_cnt <= 0) { p_err("failed to identify number of CPUs"); goto out; } + profile_cpu_id_span = profile_cpu_ids[profile_cpu_cnt - 1] + 1; profile_obj = profiler_bpf__open(); if (!profile_obj) { @@ -2539,11 +2549,12 @@ static int do_profile(int argc, char **argv) goto out; } - profile_obj->rodata->num_cpu = num_cpu; profile_obj->rodata->num_metric = num_metric; + profile_obj->rodata->cpu_id_span = profile_cpu_id_span; /* adjust map sizes */ - bpf_map__set_max_entries(profile_obj->maps.events, num_metric * num_cpu); + bpf_map__set_max_entries(profile_obj->maps.events, + num_metric * profile_cpu_id_span); bpf_map__set_max_entries(profile_obj->maps.fentry_readings, num_metric); bpf_map__set_max_entries(profile_obj->maps.accum_readings, num_metric); bpf_map__set_max_entries(profile_obj->maps.counts, 1); @@ -2589,6 +2600,10 @@ out: profiler_bpf__destroy(profile_obj); close(profile_tgt_fd); free(profile_tgt_name); + free(profile_cpu_ids); + profile_cpu_ids = NULL; + profile_cpu_cnt = 0; + profile_cpu_id_span = 0; return err; } diff --git a/tools/bpf/bpftool/skeleton/profiler.bpf.c b/tools/bpf/bpftool/skeleton/profiler.bpf.c index 6685e7252fb5..0d47a9bdc5b4 100644 --- a/tools/bpf/bpftool/skeleton/profiler.bpf.c +++ b/tools/bpf/bpftool/skeleton/profiler.bpf.c @@ -10,7 +10,7 @@ struct bpf_perf_event_value___local { __u64 running; } __attribute__((preserve_access_index)); -/* map of perf event fds, num_cpu * num_metric entries */ +/* map of perf event fds, cpu_id_span * num_metric entries */ struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(u32)); @@ -38,8 +38,8 @@ struct { __uint(value_size, sizeof(u64)); } counts SEC(".maps"); -const volatile __u32 num_cpu = 1; const volatile __u32 num_metric = 1; +const volatile __u32 cpu_id_span = 1; #define MAX_NUM_METRICS 4 SEC("fentry/XXX") @@ -60,7 +60,7 @@ int BPF_PROG(fentry_XXX) sizeof(*reading)); if (err) return 0; - key += num_cpu; + key += cpu_id_span; } return 0; @@ -100,7 +100,7 @@ int BPF_PROG(fexit_XXX) /* read all events before updating the maps, to reduce error */ for (i = 0; i < num_metric && i < MAX_NUM_METRICS; i++) { - err = bpf_perf_event_read_value(&events, cpu + i * num_cpu, + err = bpf_perf_event_read_value(&events, cpu + i * cpu_id_span, (void *)(readings + i), sizeof(*readings)); if (err) From 33c5a3278bdbef42fed02f1f4602a0367b954d03 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:18 +0100 Subject: [PATCH 250/660] btf: Extend UAPI to support BTF location (inline site) info Add BTF_KIND_LOC_PARAM, BTF_KIND_LOC_PROTO and BTF_KIND_LOCSEC to help represent location information for functions. BTF_KIND_LOC_PARAM is used to represent how we retrieve data at a location; either via register(s), or register+offset, a dereference of a register+offset or a constant value. BTF_KIND_LOC_PROTO represents location information about a location with multiple BTF_KIND_LOC_PARAMs. And finally BTF_KIND_LOCSEC is a set of location sites, each of which has - a BTF_KIND_FUNC function associated with the inline site - a location prototype specifying where to find the function parameters - an address offset relative to the kernel base address This can be used to support representing - a fully-inlined function at potentially multiple inline sites with potentially different parameter availability - a partially-inlined function where some _LOC_PROTOs represent inlined sites as above and others have normal _FUNC representations Also BTF_KIND_LOCSEC struct btf_loc will have two type id references; one for the associated func, the other for the loc_proto. Accordingly increase the number of m_offs references in btf_field_desc to 2. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924111428.75957-2-alan.maguire@oracle.com --- include/linux/btf.h | 22 ++- include/uapi/linux/btf.h | 83 ++++++++- kernel/bpf/btf.c | 322 ++++++++++++++++++++++++++++++++- tools/include/uapi/linux/btf.h | 83 ++++++++- 4 files changed, 499 insertions(+), 11 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index ddd0f4f32d24..f29ed358b7a8 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -261,6 +261,11 @@ const char *btf_type_str(const struct btf_type *t); i < btf_type_vlen(datasec_type); \ i++, member++) +#define for_each_loc(i, locsec_type, member) \ + for (i = 0, member = btf_type_loc_secinfo(locsec_type); \ + i < btf_type_vlen(locsec_type); \ + i++, member++) + static inline bool btf_type_is_ptr(const struct btf_type *t) { return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; @@ -329,6 +334,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e) return ((u64)e->val_hi32 << 32) | e->val_lo32; } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -559,7 +579,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index 618167cab4e6..11d65871d163 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h @@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4a1fa4fbdf4e..5a0179cc1676 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -346,6 +346,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; const char *btf_type_str(const struct btf_type *t) @@ -518,11 +521,27 @@ static bool btf_type_is_decl_tag(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; } +static bool btf_type_is_loc_param(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM; +} + +static bool btf_type_is_loc_proto(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO; +} + +static bool btf_type_is_locsec(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC; +} + static bool btf_type_nosize(const struct btf_type *t) { return btf_type_is_void(t) || btf_type_is_fwd(t) || btf_type_is_func(t) || btf_type_is_func_proto(t) || - btf_type_is_decl_tag(t); + btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t); } static bool btf_type_nosize_or_null(const struct btf_type *t) @@ -769,7 +788,9 @@ static bool btf_type_is_resolve_source_only(const struct btf_type *t) { return btf_type_is_var(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* What types need to be resolved? @@ -797,7 +818,9 @@ static bool btf_type_needs_resolve(const struct btf_type *t) btf_type_is_var(t) || btf_type_is_func(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* t->size can be used */ @@ -4737,6 +4760,293 @@ static const struct btf_kind_operations enum64_ops = { .show = btf_enum64_show, }; +static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 size, meta_needed, vlen = btf_vlen(t); + + meta_needed = sizeof(*p) + sizeof(__u32) * vlen; + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + size = t->size; + if (!size || size > 16) { + btf_verifier_log_type(env, t, "Unexpected size"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; +} + +static void btf_loc_param_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 i, vlen = btf_vlen(t); + + btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags); + for (i = 0; i < vlen; i++) + btf_verifier_log(env, ", %u", p->values[i]); +} + +static const struct btf_kind_operations loc_param_ops = { + .check_meta = btf_loc_param_check_meta, + .resolve = btf_df_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_param_log, + .show = btf_df_show, +}; + +static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 meta_needed; + + meta_needed = sizeof(__u32) * btf_type_vlen(t); + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; +} + +static void btf_loc_proto_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + + btf_verifier_log(env, "vlen=%u", nr_params); + for (i = 0; i < nr_params; i++, params++) + btf_verifier_log(env, ", %u", *params); +} + +static int btf_loc_proto_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + struct btf *btf = env->btf; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid loc_proto type"); + return -EINVAL; + } + + for (i = 0; i < nr_params; i++) { + const struct btf_type *param_type; + u32 param_type_id = params[i]; + + if (!param_type_id) + continue; + + param_type = btf_type_by_id(btf, param_type_id); + if (!param_type || !btf_type_is_loc_param(param_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_param#%u", i + 1); + return -EINVAL; + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations loc_proto_ops = { + .check_meta = btf_loc_proto_check_meta, + .resolve = btf_loc_proto_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_proto_log, + .show = btf_df_show, +}; + +__printf(4, 5) +static void btf_verifier_log_loc(struct btf_verifier_env *env, + const struct btf_type *locsec_type, + const struct btf_loc *loc, + const char *fmt, ...) +{ + struct bpf_verifier_log *log = &env->log; + va_list args; + + if (!bpf_verifier_log_needed(log)) + return; + if (log->level == BPF_LOG_KERNEL && !fmt) + return; + if (env->phase != CHECK_META) + btf_verifier_log_type(env, locsec_type, NULL); + + __btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u", + loc->func, loc->loc_proto, loc->offset); + if (fmt && *fmt) { + __btf_verifier_log(log, " "); + va_start(args, fmt); + bpf_verifier_vlog(log, fmt, args); + va_end(args); + } + + __btf_verifier_log(log, "\n"); +} + +static s32 btf_locsec_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 i, meta_needed, vlen = btf_type_vlen(t); + const struct btf_loc *loc; + + meta_needed = sizeof(struct btf_loc) * vlen; + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + if (!t->name_off || + !btf_name_valid_section(env->btf, t->name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + for_each_loc(i, t, loc) { + /* A loc func, loc proto cannot be in type void */ + if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) { + btf_verifier_log_loc(env, t, loc, "Invalid func"); + return -EINVAL; + } + if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) { + btf_verifier_log_loc(env, t, loc, "Invalid loc_proto"); + return -EINVAL; + } + btf_verifier_log_loc(env, t, loc, NULL); + } + + return meta_needed; +} + +static void btf_locsec_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + btf_verifier_log(env, "vlen=%u", btf_type_vlen(t)); +} + +static int btf_locsec_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const struct btf_loc *loc; + struct btf *btf = env->btf; + u32 i; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid locsec type"); + return -EINVAL; + } + + env->resolve_mode = RESOLVE_TBD; + for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i; + i < btf_type_vlen(t); i++, loc++) { + const struct btf_type *func_type, *func_proto_type; + const struct btf_type *loc_proto_type; + u32 func_type_id = loc->func; + u32 loc_proto_type_id = loc->loc_proto; + u32 proto_vlen; + + func_type = btf_type_by_id(btf, func_type_id); + if (!func_type || !btf_type_is_func(func_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + func_proto_type = btf_type_by_id(btf, func_type->type); + if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + proto_vlen = btf_vlen(func_proto_type); + + if (!env_type_is_resolved(env, func_type_id)) { + env_stack_set_next_member(env, i); + return env_stack_push(env, func_type, func_type_id); + } + + loc_proto_type = btf_type_by_id(btf, loc_proto_type_id); + if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_proto#%u", i + 1); + return -EINVAL; + } + if (proto_vlen != btf_vlen(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Mismatched vlen for loc_proto#%u", i + 1); + return -EINVAL; + } + + if (!env_type_is_resolved(env, loc_proto_type_id)) { + env_stack_set_next_member(env, i + 1); + return env_stack_push(env, loc_proto_type, + loc_proto_type_id); + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations locsec_ops = { + .check_meta = btf_locsec_check_meta, + .resolve = btf_locsec_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_locsec_log, + .show = btf_df_show, +}; + static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) @@ -5406,6 +5716,9 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = &decl_tag_ops, [BTF_KIND_TYPE_TAG] = &modifier_ops, [BTF_KIND_ENUM64] = &enum64_ops, + [BTF_KIND_LOC_PARAM] = &loc_param_ops, + [BTF_KIND_LOC_PROTO] = &loc_proto_ops, + [BTF_KIND_LOCSEC] = &locsec_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -5480,7 +5793,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env, if (!env_type_is_resolved(env, type_id)) return false; - if (btf_type_is_struct(t) || btf_type_is_datasec(t)) + if (btf_type_is_struct(t) || btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t)) return !btf_resolved_type_id(btf, type_id) && !btf_resolved_type_size(btf, type_id); diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index 618167cab4e6..11d65871d163 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h @@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ From dad2fcc644dd28977d45a8b0a1f9e02a4718cadc Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:19 +0100 Subject: [PATCH 251/660] libbpf: Add support for BTF kinds LOC[_PARAM|_PROTO|SEC] Add support for new kinds to libbpf. BTF_KIND_LOC_PARAM and BTF_KIND_LOC_PROTO are dedup-able so add support for their deduplication, whereas since BTF_KIND_LOCSEC contains a unique offset it is not. LOC_PARAM is considered a primary type since it contains no external references; LOC_PROTO is a reference type consisting of LOC_PARAM references so they are handled in the primary and reference dedup phases respectively. For BTF field iteration, BTF_KIND_LOCSEC needs 2 m_offs[] values for the associated KIND_FUNC and KIND_LOC_PROTO type ids in each LOCSEC entry. Add APIs to add location param, location prototypes and location sections and btf_is_* tests, data accessors for each. For BTF distillation we add location info to split BTF. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924111428.75957-3-alan.maguire@oracle.com --- tools/lib/bpf/btf.c | 408 +++++++++++++++++++++++++++++++- tools/lib/bpf/btf.h | 50 ++++ tools/lib/bpf/btf_dump.c | 9 + tools/lib/bpf/btf_iter.c | 18 ++ tools/lib/bpf/libbpf.c | 3 + tools/lib/bpf/libbpf.map | 6 + tools/lib/bpf/libbpf_internal.h | 2 +- 7 files changed, 491 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 908bd344229d..512b3962f75f 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -57,6 +57,9 @@ static struct btf_layout layouts[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = { sizeof(struct btf_decl_tag), 0, 0 }, [BTF_KIND_TYPE_TAG] = { 0, 0, 0 }, [BTF_KIND_ENUM64] = { 0, sizeof(struct btf_enum64), 0 }, +[BTF_KIND_LOC_PARAM] = { sizeof(struct btf_loc_param), sizeof(__u32), 0 }, +[BTF_KIND_LOC_PROTO] = { 0, sizeof(__u32), 0 }, +[BTF_KIND_LOCSEC] = { 0, sizeof(struct btf_loc), 0 }, }; struct btf { @@ -486,6 +489,12 @@ static int btf_type_size(const struct btf *btf, const struct btf_type *t) return base_size + vlen * sizeof(struct btf_var_secinfo); case BTF_KIND_DECL_TAG: return base_size + sizeof(struct btf_decl_tag); + case BTF_KIND_LOC_PARAM: + return base_size + sizeof(struct btf_loc_param) + vlen * sizeof(__u32); + case BTF_KIND_LOC_PROTO: + return base_size + vlen * sizeof(__u32); + case BTF_KIND_LOCSEC: + return base_size + vlen * sizeof(struct btf_loc); default: return btf_type_size_unknown(btf, t); } @@ -501,12 +510,15 @@ static void btf_bswap_type_base(struct btf_type *t) static int btf_bswap_type_rest(struct btf_type *t) { struct btf_var_secinfo *v; + struct btf_loc_param *lp; struct btf_enum64 *e64; struct btf_member *m; struct btf_array *a; struct btf_param *p; struct btf_enum *e; + struct btf_loc *l; __u32 vlen = btf_vlen(t); + __u32 *d; int i; switch (btf_kind(t)) { @@ -569,6 +581,23 @@ static int btf_bswap_type_rest(struct btf_type *t) case BTF_KIND_DECL_TAG: btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx); return 0; + case BTF_KIND_LOC_PARAM: + lp = btf_loc_param(t); + lp->flags = bswap_32(lp->flags); + for (i = 0, d = (__u32 *)(lp + 1); i < vlen; i++, d++) + *d = bswap_32(*d); + return 0; + case BTF_KIND_LOC_PROTO: + for (i = 0, d = btf_loc_proto_params(t); i < vlen; i++, d++) + *d = bswap_32(*d); + return 0; + case BTF_KIND_LOCSEC: + for (i = 0, l = btf_locsec_locs(t); i < vlen; i++, l++) { + l->func = bswap_32(l->func); + l->loc_proto = bswap_32(l->loc_proto); + l->offset = bswap_32(l->offset); + } + return 0; default: pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); return -EINVAL; @@ -745,6 +774,33 @@ static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __ } break; } + case BTF_KIND_LOC_PARAM: + break; + case BTF_KIND_LOC_PROTO: { + __u32 *p = btf_loc_proto_params(t); + + n = btf_vlen(t); + for (i = 0; i < n; i++, p++) { + err = btf_validate_id(btf, *p, id); + if (err) + return err; + } + break; + } + case BTF_KIND_LOCSEC: { + const struct btf_loc *l = btf_locsec_locs(t); + + n = btf_vlen(t); + for (i = 0; i < n; i++, l++) { + if (!err) + err = btf_validate_id(btf, l->func, id); + if (!err) + err = btf_validate_id(btf, l->loc_proto, id); + if (err) + return err; + } + break; + } default: /* Kind may be represented in kind layout information. */ if (btf_type_size_unknown(btf, t) < 0) { @@ -3344,6 +3400,243 @@ int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1); } +/* + * Append new BTF_KIND_LOC_PARAM with specified size and flags. Values are + * added via btf__add_loc_param_value(). + * + * - *byte_sz* - size of the type, in bytes (1 through 16); + * - *flags* - combination of enum btf_loc_param_flags values + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_loc_param(struct btf *btf, __u32 byte_sz, __u32 flags) +{ + struct btf_loc_param *p; + struct btf_type *t; + int sz, err; + + if (!byte_sz || byte_sz > 16) + return libbpf_err(-EINVAL); + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(struct btf_type) + sizeof(*p); + t = btf_add_type_mem(btf, sz); + if (!t) + return libbpf_err(-ENOMEM); + + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_LOC_PARAM, 0, 0); + t->size = byte_sz; + + p = btf_loc_param(t); + p->flags = flags; + + return btf_commit_type(btf, sz); +} + +/* + * Append *value* to existing BTF_KIND_LOC_PARAM. + * + * Returns: + * - 0 on success + * - <0, on error. + */ +int btf__add_loc_param_value(struct btf *btf, __u32 value) +{ + struct btf_type *t; + int sz, err; + __u32 *v; + + /* last type should be BTF_KIND_LOC_PARAM */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_loc_param(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(value); + v = btf_add_type_mem(btf, sz); + if (!v) + return libbpf_err(-ENOMEM); + *v = value; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + +/* + * Append new BTF_KIND_LOC_PROTO + * + * The prototype is then populated with 0 or more BTF_KIND_LOC_PARAMs via + * btf__add_loc_proto_param(); similar to how btf__add_func_param() adds + * parameters to a FUNC_PROTO. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_loc_proto(struct btf *btf) +{ + struct btf_type *t; + int err; + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + t = btf_add_type_mem(btf, sizeof(struct btf_type)); + if (!t) + return libbpf_err(-ENOMEM); + + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_LOC_PROTO, 0, 0); + t->size = 0; + + return btf_commit_type(btf, sizeof(struct btf_type)); +} + +/* + * Append a BTF_KIND_LOC_PARAM id/0 to BTF_KIND_LOC_PROTO. + * + * Returns: + * - 0 on success; + * - <0, on error. + + */ +int btf__add_loc_proto_param(struct btf *btf, __u32 id) +{ + struct btf_type *t; + int sz, err; + __u32 *p; + + if (validate_type_id(id)) + return libbpf_err(-EINVAL); + + /* last type should be BTF_KIND_LOC_PROTO */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_loc_proto(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(__u32); + p = btf_add_type_mem(btf, sz); + if (!p) + return libbpf_err(-ENOMEM); + *p = id; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + +/* + * Append new BTF_KIND_LOCSEC type with: + * - *name* - non-empty/non-NULL name; + * + * Location section is initially empty. Location info can be added with + * btf__add_locsec_loc() calls, after btf__add_locsec() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ + +int btf__add_locsec(struct btf *btf, const char *name) +{ + struct btf_type *t; + int name_off = 0; + int err; + + /* non-empty name */ + if (str_is_empty(name)) + return libbpf_err(-EINVAL); + + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + t = btf_add_type_mem(btf, sizeof(struct btf_type)); + if (!t) + return libbpf_err(-ENOMEM); + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return libbpf_err(name_off); + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_LOCSEC, 0, 0); + t->size = 0; + + return btf_commit_type(btf, sizeof(struct btf_type)); +} + +int btf__add_locsec_loc(struct btf *btf, __u32 func, __u32 loc_proto, + __u32 offset) +{ + struct btf_type *t; + struct btf_loc *l; + int sz, err; + + if (validate_type_id(func) || validate_type_id(loc_proto)) + return libbpf_err(-EINVAL); + + /* last type should be BTF_KIND_LOCSEC */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_locsec(t)) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + err = btf_ensure_modifiable(btf); + if (err) + return libbpf_err(err); + + sz = sizeof(*l); + l = btf_add_type_mem(btf, sz); + if (!l) + return libbpf_err(-ENOMEM); + + l->func = func; + l->loc_proto = loc_proto; + l->offset = offset; + + /* update parent type's vlen */ + t = btf_last_type(btf); + err = btf_type_inc_vlen(t); + if (err) + return libbpf_err(err); + + btf_hdr_update_type_len(btf, btf->hdr.type_len + sz); + return 0; +} + struct btf_ext_sec_info_param { __u32 off; __u32 len; @@ -4114,8 +4407,8 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_o for (i = 1; i < type_cnt; i++) { struct btf_type *t = btf_type_by_id(d->btf, i); - /* VAR and DATASEC are never deduped and are self-canonical */ - if (btf_is_var(t) || btf_is_datasec(t)) + /* VAR, DATASEC and LOCSEC are never deduped and are self-canonical */ + if (btf_is_var(t) || btf_is_datasec(t) || btf_is_locsec(t)) d->map[i] = i; else d->map[i] = BTF_UNPROCESSED_ID; @@ -4395,6 +4688,47 @@ static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) btf_is_any_enum(t1) && btf_is_any_enum(t2); } +static long btf_hash_loc_param(struct btf_type *t) +{ + struct btf_loc_param *p = btf_loc_param(t); + long h = btf_hash_common(t); + int i, vlen = btf_vlen(t); + + h = hash_combine(h, p->flags); + + for (i = 0; i < vlen; i++) + h = hash_combine(h, p->values[i]); + return h; +} + +static long btf_hash_loc_proto(struct btf_type *t) +{ + __u32 *p = btf_loc_proto_params(t); + long h = btf_hash_common(t); + int i, vlen = btf_vlen(t); + + for (i = 0; i < vlen; i++, p++) + h = hash_combine(h, *p); + return h; +} + +static bool btf_equal_loc_param(struct btf_type *t1, struct btf_type *t2) +{ + struct btf_loc_param *p1 = btf_loc_param(t1); + struct btf_loc_param *p2 = btf_loc_param(t2); + int i, vlen = btf_vlen(t1); + + if (!btf_equal_common(t1, t2)) + return false; + if (p1->flags != p2->flags) + return false; + for (i = 0; i < vlen; i++) { + if (p1->values[i] != p2->values[i]) + return false; + } + return true; +} + /* * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, * as referenced type IDs equivalence is established separately during type @@ -4589,7 +4923,8 @@ static int btf_dedup_prep(struct btf_dedup *d) switch (btf_kind(t)) { case BTF_KIND_VAR: case BTF_KIND_DATASEC: - /* VAR and DATASEC are never hash/deduplicated */ + case BTF_KIND_LOCSEC: + /* VAR DATASEC and LOCSEC are never hash/deduplicated */ continue; case BTF_KIND_CONST: case BTF_KIND_VOLATILE: @@ -4622,6 +4957,12 @@ static int btf_dedup_prep(struct btf_dedup *d) case BTF_KIND_FUNC_PROTO: h = btf_hash_fnproto(t); break; + case BTF_KIND_LOC_PARAM: + h = btf_hash_loc_param(t); + break; + case BTF_KIND_LOC_PROTO: + h = btf_hash_loc_proto(t); + break; default: pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id); return -EINVAL; @@ -4664,6 +5005,8 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_DATASEC: case BTF_KIND_DECL_TAG: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: return 0; case BTF_KIND_INT: @@ -4713,6 +5056,18 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) } break; + case BTF_KIND_LOC_PARAM: + h = btf_hash_loc_param(t); + for_each_dedup_cand(d, hash_entry, h) { + cand_id = hash_entry->value; + cand = btf_type_by_id(d->btf, cand_id); + if (btf_equal_loc_param(t, cand)) { + new_id = cand_id; + break; + } + } + break; + default: return -EINVAL; } @@ -5145,6 +5500,13 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, return 1; } + case BTF_KIND_LOC_PARAM: + return btf_equal_loc_param(cand_type, canon_type); + + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + return 0; + default: return -EINVAL; } @@ -5489,6 +5851,37 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) break; } + case BTF_KIND_LOC_PROTO: { + __u32 *p1, *p2; + __u32 i, vlen; + + p1 = btf_loc_proto_params(t); + vlen = btf_vlen(t); + + for (i = 0; i < vlen; i++, p1++) { + ref_type_id = btf_dedup_ref_type(d, *p1); + if (ref_type_id < 0) + return ref_type_id; + *p1 = ref_type_id; + } + + h = btf_hash_loc_proto(t); + for_each_dedup_cand(d, hash_entry, h) { + cand_id = hash_entry->value; + cand = btf_type_by_id(d->btf, cand_id); + if (!btf_equal_common(t, cand)) + continue; + vlen = btf_vlen(cand); + p1 = btf_loc_proto_params(t); + p2 = btf_loc_proto_params(cand); + if (memcmp(p1, p2, vlen * sizeof(__u32)) == 0) { + new_id = cand_id; + break; + } + } + break; + } + default: return -EINVAL; } @@ -5970,8 +6363,11 @@ static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_VOLATILE: + case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: dist->id_map[*id] = *id; break; default: @@ -5997,7 +6393,7 @@ static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i) static int btf_add_distilled_types(struct btf_distill *dist) { - bool adding_to_base = dist->pipe.dst->start_id == 1; + bool adding_to_base = dist->pipe.dst->base_btf == NULL; int id = btf__type_cnt(dist->pipe.dst); struct btf_type *t; int i, err = 0; @@ -6065,8 +6461,12 @@ static int btf_add_distilled_types(struct btf_distill *dist) case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_VOLATILE: + case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_TYPE_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: /* All other types are added to split BTF. */ if (adding_to_base) continue; diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 587172c0de08..57f12630c5d1 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -274,6 +274,20 @@ LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_typ LIBBPF_API int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id, int component_idx); +/* location construction APIs */ +LIBBPF_API int btf__add_loc_param(struct btf *btf, __u32 size, __u32 flags); + +LIBBPF_API int btf__add_loc_param_value(struct btf *btf, __u32 value); + +LIBBPF_API int btf__add_loc_proto(struct btf *btf); + +LIBBPF_API int btf__add_loc_proto_param(struct btf *btf, __u32 id); + +LIBBPF_API int btf__add_locsec(struct btf *btf, const char *name); + +LIBBPF_API int btf__add_locsec_loc(struct btf *btf, __u32 func, __u32 loc_proto, + __u32 offset); + struct btf_dedup_opts { size_t sz; /* optional .BTF.ext info to dedup along the main BTF info */ @@ -431,6 +445,12 @@ btf_dump__dump_type_data(struct btf_dump *d, __u32 id, #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ #define BTF_KIND_ENUM64 19 /* Enum for up-to 64bit values */ +#define BTF_KIND_LOC_PARAM 20 /* Parameter at location */ +#define BTF_KIND_LOC_PROTO 21 /* Parameter set at location */ +#define BTF_KIND_LOCSEC 22 /* Section containing location info */ + +struct btf_loc_param; +struct btf_loc; static inline __u16 btf_kind(const struct btf_type *t) { @@ -569,6 +589,21 @@ static inline bool btf_is_any_enum(const struct btf_type *t) return btf_is_enum(t) || btf_is_enum64(t); } +static inline bool btf_is_loc_param(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOC_PARAM; +} + +static inline bool btf_is_loc_proto(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOC_PROTO; +} + +static inline bool btf_is_locsec(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_LOCSEC; +} + static inline bool btf_kind_core_compat(const struct btf_type *t1, const struct btf_type *t2) { @@ -683,6 +718,21 @@ static inline struct btf_decl_tag *btf_decl_tag(const struct btf_type *t) return (struct btf_decl_tag *)(t + 1); } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_locsec_locs(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 123c448f20c7..fa995c02a170 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -328,6 +328,9 @@ static int btf_dump_mark_referenced(struct btf_dump *d) case BTF_KIND_ENUM64: case BTF_KIND_FWD: case BTF_KIND_FLOAT: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: break; case BTF_KIND_VOLATILE: @@ -609,6 +612,9 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) case BTF_KIND_VAR: case BTF_KIND_DATASEC: case BTF_KIND_DECL_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: d->type_states[id].order_state = ORDERED; return 0; @@ -2525,6 +2531,9 @@ static int btf_dump_dump_type_data(struct btf_dump *d, case BTF_KIND_FUNC: case BTF_KIND_FUNC_PROTO: case BTF_KIND_DECL_TAG: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: err = btf_dump_unsupported_data(d, t, id); break; case BTF_KIND_INT: diff --git a/tools/lib/bpf/btf_iter.c b/tools/lib/bpf/btf_iter.c index 9a6c822c2294..199063b55ebe 100644 --- a/tools/lib/bpf/btf_iter.c +++ b/tools/lib/bpf/btf_iter.c @@ -29,6 +29,7 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, case BTF_KIND_FLOAT: case BTF_KIND_ENUM: case BTF_KIND_ENUM64: + case BTF_KIND_LOC_PARAM: it->desc = (struct btf_field_desc) {}; break; case BTF_KIND_FWD: @@ -71,6 +72,20 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, 1, {offsetof(struct btf_var_secinfo, type)} }; break; + case BTF_KIND_LOC_PROTO: + it->desc = (struct btf_field_desc) { + 0, {}, + sizeof(__u32), + 1, {0}}; + break; + case BTF_KIND_LOCSEC: + it->desc = (struct btf_field_desc) { + 0, {}, + sizeof(struct btf_loc), + 2, {offsetof(struct btf_loc, func), + offsetof(struct btf_loc, loc_proto)}}; + break; + default: return -EINVAL; } @@ -94,6 +109,9 @@ int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, case BTF_KIND_DECL_TAG: case BTF_KIND_TYPE_TAG: case BTF_KIND_DATASEC: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: it->desc = (struct btf_field_desc) { 1, {offsetof(struct btf_type, name_off)} }; diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 39fc3c5e6bd2..665b88c84c1c 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -2524,6 +2524,9 @@ static const char *__btf_kind_str(__u16 kind) case BTF_KIND_DECL_TAG: return "decl_tag"; case BTF_KIND_TYPE_TAG: return "type_tag"; case BTF_KIND_ENUM64: return "enum64"; + case BTF_KIND_LOC_PARAM: return "loc_param"; + case BTF_KIND_LOC_PROTO: return "loc_proto"; + case BTF_KIND_LOCSEC: return "locsec"; default: return "unknown"; } } diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 7a84dd00ce95..18d27d20102e 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -463,6 +463,12 @@ LIBBPF_1.8.0 { bpf_program__attach_tracing_multi; bpf_program__clear_flags; bpf_program__clone; + btf__add_loc_param; + btf__add_loc_param_value; + btf__add_loc_proto; + btf__add_loc_proto_param; + btf__add_locsec; + btf__add_locsec_loc; btf__find_by_name_kind_own; btf__new_empty_opts; } LIBBPF_1.7.0; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index cb4d96233844..546f65b95cf4 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -580,7 +580,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { From 0f1de9408e5909709d95c6352c624db4e725722c Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:20 +0100 Subject: [PATCH 252/660] selftests/bpf: Test helper support for BTF_KIND_LOC[_PARAM|_PROTO|SEC] Add support to dump and validate new location-related kinds. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924111428.75957-4-alan.maguire@oracle.com --- tools/testing/selftests/bpf/btf_helpers.c | 35 ++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c index 1c1c2c26690a..85a4d15d1dd1 100644 --- a/tools/testing/selftests/bpf/btf_helpers.c +++ b/tools/testing/selftests/bpf/btf_helpers.c @@ -27,11 +27,14 @@ static const char * const btf_kind_str_mapping[] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; static const char *btf_kind_str(__u16 kind) { - if (kind > BTF_KIND_ENUM64) + if (kind > BTF_KIND_LOCSEC) return "UNKNOWN"; return btf_kind_str_mapping[kind]; } @@ -203,6 +206,36 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) fprintf(out, " type_id=%u component_idx=%d", t->type, btf_decl_tag(t)->component_idx); break; + case BTF_KIND_LOC_PARAM: { + struct btf_loc_param *p = btf_loc_param(t); + + fprintf(out, " size=%u flags=0x%x vlen=%u", t->size, p->flags, vlen); + for (i = 0; i < vlen; i++) { + if (p->flags & BTF_LOC_PARAM_SIGNED) + fprintf(out, "\n\tvalue=%d", (__s32)p->values[i]); + else + fprintf(out, "\n\tvalue=%u", p->values[i]); + } + break; + } + case BTF_KIND_LOC_PROTO: { + const __u32 *p = btf_loc_proto_params(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, p++) + fprintf(out, "\n\ttype_id=%u", *p); + break; + } + case BTF_KIND_LOCSEC: { + const struct btf_loc *l = btf_locsec_locs(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, l++) { + fprintf(out, "\n\tfunc_type_id=%u loc_proto_type_id=%u offset=%u", + l->func, l->loc_proto, l->offset); + } + break; + } default: break; } From 663fa771389d8cdab53283abb6985984628defa1 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:21 +0100 Subject: [PATCH 253/660] selftests/bpf: Add LOC_PARAM, LOC_PROTO, LOCSEC to field iter tests BTF_KIND_LOC[_PARAM|_PROTO|SEC] need to work with field iteration, so extend the selftest to cover these and ensure iteration over all types and names succeeds. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924111428.75957-5-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/btf_field_iter.c | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c index 32159d3eb281..dcb5429d141d 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c @@ -31,8 +31,11 @@ struct field_data { { .ids = { 11 }, .strs = { "decltag" } }, { .ids = { 6 }, .strs = { "typetag" } }, { .ids = {}, .strs = { "e64", "eval1", "eval2", "eval3" } }, - { .ids = { 15, 16 }, .strs = { "datasec1" } } - + { .ids = { 15, 16 }, .strs = { "datasec1" } }, + { .ids = {}, .strs = { "" } }, + { .ids = {}, .strs = { "" } }, + { .ids = { 22, 23 }, .strs = { "" } }, + { .ids = { 14, 24 }, .strs = { ".loc" } } }; /* Fabricate BTF with various types and check BTF field iteration finds types, @@ -88,6 +91,19 @@ void test_btf_field_iter(void) btf__add_datasec_var_info(btf, 15, 0, 4); btf__add_datasec_var_info(btf, 16, 4, 8); + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED); + /* [22] loc value -1 */ + btf__add_loc_param_value(btf, -1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG); /* [23] loc reg 1 */ + btf__add_loc_param_value(btf, 1); + + btf__add_loc_proto(btf); /* [24] loc proto */ + btf__add_loc_proto_param(btf, 22); /* param value -1, */ + btf__add_loc_proto_param(btf, 23); /* param reg 1 */ + + btf__add_locsec(btf, ".loc"); /* [25] locsec ".loc" */ + btf__add_locsec_loc(btf, 14, 24, 128); /* "func" */ + VALIDATE_RAW_BTF( btf, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", @@ -123,7 +139,16 @@ void test_btf_field_iter(void) "\t'eval3' val=3000", "[21] DATASEC 'datasec1' size=12 vlen=2\n" "\ttype_id=15 offset=0 size=4\n" - "\ttype_id=16 offset=4 size=8"); + "\ttype_id=16 offset=4 size=8", + "[22] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[23] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[24] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=22\n" + "\ttype_id=23", + "[25] LOCSEC '.loc' vlen=1\n" + "\tfunc_type_id=14 loc_proto_type_id=24 offset=128"); for (id = 1; id < btf__type_cnt(btf); id++) { struct btf_type *t = btf_type_by_id(btf, id); From dfb463b07117ca09552ff63bc94ea549884ec04f Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:22 +0100 Subject: [PATCH 254/660] selftests/bpf: Add LOC_PARAM, LOC_PROTO, LOCSEC to dedup split tests Ensure that location params/protos are deduplicated and location sections are not, and that references to deduplicated locations within location prototypes and sections are updated after deduplication. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924111428.75957-6-alan.maguire@oracle.com --- .../bpf/prog_tests/btf_dedup_split.c | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 9d6161151593..5c15a6b7f4fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c @@ -554,6 +554,120 @@ cleanup: btf__free(vmlinux_btf); } +static void test_split_loc(void) +{ + struct btf *btf1, *btf2; + int err; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */ + + + btf__add_int(btf1, "long", 8, BTF_INT_SIGNED); /* [1] long */ + btf__add_ptr(btf1, 1); /* [2] ptr to long */ + btf__add_func_proto(btf1, 1); /* [3] long (*)(long, long *); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func_param(btf1, "p2", 2); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 3); /* [4] long foo(long, long *); */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, 3735928559); + btf__add_loc_param_value(btf1, 4277009102); /* [5] loc value */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_REG); /* [6] loc reg 1 */ + btf__add_loc_param_value(btf1, 1); + btf__add_loc_proto(btf1); /* [7] loc proto */ + btf__add_loc_proto_param(btf1, 5); /* param value */ + btf__add_loc_proto_param(btf1, 6); /* param reg 1 */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + btf__add_loc_param(btf2, 8, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf2, 1); /* [8] loc reg 1 */ + btf__add_loc_proto(btf2); /* [9] loc proto */ + btf__add_loc_proto_param(btf2, 5); /* param value */ + btf__add_loc_proto_param(btf2, 8); /* param reg 1 */ + btf__add_locsec(btf2, "inline.text"); /* [10] locsec "inline.text" */ + /* add duplicate locsec */ + btf__add_locsec_loc(btf2, 4, 9, 128); + btf__add_locsec(btf2, "inline.text"); /* [11] locsec "inline.text" */ + btf__add_locsec_loc(btf2, 4, 9, 128); + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[9] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=8", + "[10] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128", + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128"); + + err = btf__dedup(btf2, NULL); + if (!ASSERT_OK(err, "btf_dedup")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128", + "[9] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128"); + +cleanup: + btf__free(btf2); + btf__free(btf1); +} + void test_btf_dedup_split() { if (test__start_subtest("split_simple")) @@ -566,4 +680,6 @@ void test_btf_dedup_split() test_split_dup_struct_in_cu(); if (test__start_subtest("split_module")) test_split_module(); + if (test__start_subtest("split_loc")) + test_split_loc(); } From 49a4c23a22347df28f14d79636b456c61c0fcf70 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:23 +0100 Subject: [PATCH 255/660] selftests/bpf: BTF distill tests to ensure LOC[_PARAM|_PROTO] add to split BTF When creating distilled BTF, BTF_KIND_FUNC, _LOC_PARAM and _LOC_PROTO should be added to split BTF. This means potentially some duplication of location information, but only for out-of-tree modules that use distilled base/split BTF. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-7-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/btf_distill.c | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_distill.c b/tools/testing/selftests/bpf/prog_tests/btf_distill.c index fb67ae195a73..f0900b47fd48 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_distill.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_distill.c @@ -671,6 +671,110 @@ cleanup: btf__free(btf1); } +/* LOC_PARAM, LOC_PROTO should be added to split BTF. */ +static void test_distilled_loc(void) +{ + struct btf *btf1 = NULL, *btf2 = NULL, *btf3 = NULL; + struct btf *btf4 = NULL, *btf5 = NULL; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_func_proto(btf1, 1); /* [2] int (*)(int); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 2); /* [3] int foo(int); */ + btf__add_loc_param(btf1, 4, BTF_LOC_PARAM_SIGNED | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, -1); /* [4] loc value */ + btf__add_loc_proto(btf1); /* [5] loc proto */ + btf__add_loc_proto_param(btf1, 4); /* param value */ + btf__add_int(btf1, "unsigned int", 4, 0); /* [6] unsigned int */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + + btf__add_locsec(btf2, "inline.text"); /* [6] locsec */ + btf__add_locsec_loc(btf2, 3, 5, 256); /* "foo" offset 256 */ + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)", + "[7] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=3 loc_proto_type_id=5 offset=256"); + + if (!ASSERT_EQ(0, btf__distill_base(btf2, &btf3, &btf4), + "distilled_base") || + !ASSERT_OK_PTR(btf3, "distilled_base") || + !ASSERT_OK_PTR(btf4, "distilled_split") || + !ASSERT_EQ(2, btf__type_cnt(btf3), "distilled_base_type_cnt")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + /* remainder is split BTF */ + "[2] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=6 offset=256", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[6] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=5"); + + btf5 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) + goto cleanup; + btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_int(btf5, "char", 1, 0); /* [2] char */ + + if (!ASSERT_EQ(btf__relocate(btf4, btf5), 0, "relocate_split")) + goto cleanup; + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'char' size=1 bits_offset=0 nr_bits=8 encoding=(none)", + /* remainder is split BTF */ + "[3] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=5 loc_proto_type_id=7 offset=256", + "[4] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[5] FUNC 'foo' type_id=4 linkage=static", + "[6] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[7] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=6"); + +cleanup: + btf__free(btf5); + btf__free(btf4); + btf__free(btf3); + btf__free(btf2); + btf__free(btf1); +} + void test_btf_distill(void) { if (test__start_subtest("distilled_base")) @@ -689,4 +793,6 @@ void test_btf_distill(void) test_distilled_base_vmlinux(); if (test__start_subtest("distilled_endianness")) test_distilled_endianness(); + if (test__start_subtest("distilled_loc")) + test_distilled_loc(); } From 7890a57356a2103f7cc79edcd87fb8ed7c23bdcc Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:24 +0100 Subject: [PATCH 256/660] bpftool: Handle multi-split BTF by supporting multiple base BTFs For bpftool to be able to dump .BTF.inline data in /sys/kernel/btf/foo.inline for module foo, it needs to support multi-split BTF because the parent-child relationship of BTF inline data for modules is vmlinux BTF data module BTF data module BTF inline data So for example to dump BTF inline info for xfs we would run $ bpftool btf dump -B /sys/kernel/btf/vmlinux -B /sys/kernel/btf/xfs file /sys/kernel/btf/xfs.inline Multiple bases are specified with the vmlinux base BTF first (parent) followed by the xfs BTF (child), and finally the XFS BTF inline info. Update help text accordingly to reflect the ability to specify multiple in-order root-to-branch bases. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-8-alan.maguire@oracle.com --- tools/bpf/bpftool/btf.c | 2 +- tools/bpf/bpftool/main.c | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index bca0a3982f09..65e8a29277e6 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -1545,7 +1545,7 @@ static int do_help(int argc, char **argv) " " HELP_SPEC_MAP "\n" " " HELP_SPEC_PROGRAM "\n" " " HELP_SPEC_OPTIONS " |\n" - " {-B|--base-btf} }\n" + " {[{-B|--base-btf} FILE]... }}\n" "", bin_name, "btf"); diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 5ababd8f7d0a..83884b21e708 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -466,6 +466,7 @@ int main(int argc, char **argv) { "base-btf", required_argument, NULL, 'B' }, { 0 } }; + struct btf *new_base_btf = NULL, *root_base_btf = NULL; bool version_requested = false; int opt, ret; @@ -515,12 +516,16 @@ int main(int argc, char **argv) verifier_logs = true; break; case 'B': - base_btf = btf__parse(optarg, NULL); - if (!base_btf) { + /* handle multi-split BTF */ + new_base_btf = btf__parse_split(optarg, base_btf); + if (!new_base_btf) { p_err("failed to parse base BTF at '%s': %d\n", optarg, -errno); return -1; } + base_btf = new_base_btf; + if (!root_base_btf) + root_base_btf = base_btf; break; case 'L': use_loader = true; @@ -567,7 +572,20 @@ int main(int argc, char **argv) if (json_output) jsonw_destroy(&json_wtr); - btf__free(base_btf); + while (base_btf) { + bool is_root = base_btf == root_base_btf; + + new_base_btf = (struct btf *)btf__base_btf(base_btf); + btf__free(base_btf); + /* + * Do not free base BTF that is an owned .BTF.base ; leads + * to a double-free, so only free as far as the root base + * we explicitly read with -B above. + */ + if (is_root) + break; + base_btf = new_base_btf; + } return ret; } From 0ec4caf73ce55d0aa99f78c0fa7d5d8830e0ecd1 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:25 +0100 Subject: [PATCH 257/660] bpftool: Document support for multi-split BTF Document the ability to pass multiple levels of split BTF, using "-B base-btf" options. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-9-alan.maguire@oracle.com --- tools/bpf/bpftool/Documentation/bpftool-btf.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-btf.rst b/tools/bpf/bpftool/Documentation/bpftool-btf.rst index cf75a7fa2d6b..c6dc445cd4a1 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-btf.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-btf.rst @@ -16,7 +16,7 @@ SYNOPSIS **bpftool** [*OPTIONS*] **btf** *COMMAND* -*OPTIONS* := { |COMMON_OPTIONS| | { **-B** | **--base-btf** } } +*OPTIONS* := { |COMMON_OPTIONS| | [{ **-B** | **--base-btf**} *FILE*]... } *COMMANDS* := { **dump** | **help** } @@ -87,7 +87,10 @@ OPTIONS objects for kernel modules. To avoid duplicating all kernel symbols required by modules, BTF objects for modules are "split", they are built incrementally on top of the kernel (vmlinux) BTF object. So the - base BTF reference should usually point to the kernel BTF. + base BTF reference should usually point to the kernel BTF. Multiple + base BTF objects can be passed, where the first is assumed to be the + root BTF, followed by split BTF based upon it, followed by split + BTF based upon the first split BTF and so on. When the main BTF object to process (for example, the module BTF to dump) is passed as a *FILE*, bpftool attempts to autodetect the path From 321562c34d5b19796d7d0b82ff731d9f66a3453a Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:26 +0100 Subject: [PATCH 258/660] bpftool: Add ability to dump LOC_PARAM, LOC_PROTO and LOCSEC In raw mode ensure we can dump new BTF kinds in normal/json format. BTF_KIND_LOC_PARAMs are rendered as strings, for example a const value of 0x2a and a dereference of %r10 + 0x20: [12] LOC_PARAM '(anon)' size=4 flags=0x2 vlen=1 values='0x2a' [13] LOC_PARAM '(anon)' size=8 flags=0x38 vlen=2 values='*(reg10 + 0x20)' LOC_PROTOs render the associated values of each of their LOC_PARAMs for easier readability: [14] LOC_PROTO '(anon)' vlen=2 type_id=12 value='0x2a' type_id=13 value='*(reg10 + 0x20)' and LOCSEC shows function name associated with site: [15] LOCSEC 'inline.text' vlen=1 name='foo' func_type_id=5 loc_proto_type_id=14 offset=64 Registers are displayed in an architecture-neutral form; 'regN' where N is the DWARF register number, or 'fbreg' for the stack frame base. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-10-alan.maguire@oracle.com --- tools/bpf/bpftool/btf.c | 224 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 65e8a29277e6..e29c8a84e224 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -29,6 +29,7 @@ #define MAX_ROOT_IDS 16 #define MAX_BTF_FILES 64 +#define MAX_LOC_PARAM_WORDS 8 static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_UNKN] = "UNKNOWN", @@ -51,6 +52,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; struct sort_datum { @@ -117,6 +121,137 @@ static int btf_kind_safe(int kind) return kind <= BTF_KIND_MAX ? kind : BTF_KIND_UNKN; } +static void btf_loc_param_reg_str(__u32 reg, char *str, size_t sz) +{ + if (reg == BTF_LOC_PARAM_FBREG) + snprintf(str, sz, "fbreg"); + else + snprintf(str, sz, "reg%u", reg); +} + +static void btf_loc_param_raw_str(const struct btf_loc_param *p, __u32 vlen, + char *str, size_t sz) +{ + __u32 i, nr_words = min(vlen, (__u32)MAX_LOC_PARAM_WORDS); + size_t off = 0; + + if (!sz) + return; + + off += snprintf(str + off, sz - off, "raw=["); + for (i = 0; i < nr_words && off < sz; i++) + off += snprintf(str + off, sz - off, "%s0x%08x", + i ? ", " : "", p->values[i]); + if (vlen > nr_words && off < sz) + off += snprintf(str + off, sz - off, ", ..."); + if (off < sz) + snprintf(str + off, sz - off, "]"); +} + +static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) +{ + const struct btf_loc_param *p; + __u32 i = 0, vlen; + __u64 value; + __u32 value_size; + bool negative = false; + char regs[32] = {}; + char num[32] = {}; + const char *op = ""; + + if (!t || !btf_is_loc_param(t)) { + snprintf(str, sz, ""); + return; + } + + p = btf_loc_param(t); + vlen = btf_vlen(t); + + if (p->flags & BTF_LOC_PARAM_REG) { + __u32 nregs = (p->flags == BTF_LOC_PARAM_REG) ? vlen : 1; + + if (nregs > vlen) { + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + + switch (nregs) { + case 2: + btf_loc_param_reg_str(p->values[0], regs, sizeof(regs)); + snprintf(regs + strlen(regs), sizeof(regs) - strlen(regs), + ", "); + btf_loc_param_reg_str(p->values[1], + regs + strlen(regs), + sizeof(regs) - strlen(regs)); + break; + case 1: + btf_loc_param_reg_str(p->values[0], regs, sizeof(regs)); + break; + default: + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + i += nregs; + } + if (p->flags & (BTF_LOC_PARAM_CONST|BTF_LOC_PARAM_OFFSET)) { + switch (vlen - i) { + case 1: + value_size = sizeof(p->values[0]); + value = p->values[i]; + break; + case 2: + value_size = 2 * sizeof(p->values[0]); + value = ((__u64)p->values[i + 1] << 32) | p->values[i]; + break; + default: + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + i = vlen; + if (p->flags & BTF_LOC_PARAM_SIGNED) { + /* + * size describes the represented parameter, so it + * describes a constant's signed width. An offset + * is determined by its value words after the register + * number. + */ + __u32 size = p->flags & BTF_LOC_PARAM_OFFSET ? + value_size : t->size; + __u32 bits = size * 8; + + /* + * Since we represent constant values in hex, we + * need to determine if the value is negative so + * we can prepend a "-", and also fix the value + * to be positive so we can have - 0x. + */ + if (size && size <= sizeof(value)) { + if (size < sizeof(value)) + value &= (1ULL << bits) - 1; + negative = value & (1ULL << (bits - 1)); + if (negative) + value = size == sizeof(value) ? -value : + (1ULL << bits) - value; + } + } + snprintf(num, sizeof(num), "0x%llx%s", (unsigned long long)value, + p->flags & BTF_LOC_PARAM_ADDR ? " (addr)" : ""); + } + if (i != vlen) { + btf_loc_param_raw_str(p, vlen, str, sz); + return; + } + if (num[0]) + op = regs[0] ? (negative ? " - " : " + ") : negative ? "-" : ""; + + snprintf(str, sz, "%s%s%s%s%s", + p->flags & BTF_LOC_PARAM_DEREF ? "*(" : "", + regs, + op, + num, + p->flags & BTF_LOC_PARAM_DEREF ? ")" : ""); +} + static int dump_btf_type(const struct btf *btf, __u32 id, const struct btf_type *t) { @@ -415,6 +550,95 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } break; } + case BTF_KIND_LOC_PARAM: { + const struct btf_loc_param *p = btf_loc_param(t); + __u32 vlen = btf_vlen(t); + char param_str[256] = {}; + + btf_loc_param_str(t, param_str, sizeof(param_str)); + + if (json_output) { + jsonw_uint_field(w, "size", t->size); + jsonw_uint_field(w, "flags", p->flags); + jsonw_uint_field(w, "vlen", vlen); + jsonw_string_field(w, "values", param_str); + } else { + printf(" size=%u flags=0x%x vlen=%u values='%s'", + t->size, p->flags, vlen, param_str); + } + break; + } + case BTF_KIND_LOC_PROTO: { + __u32 *params = btf_loc_proto_params(t); + __u32 i, vlen = btf_vlen(t); + + if (json_output) { + jsonw_uint_field(w, "vlen", vlen); + jsonw_name(w, "params"); + jsonw_start_array(w); + } else { + printf(" vlen=%u", vlen); + } + + for (i = 0; i < vlen; i++, params++) { + const struct btf_type *p; + char param_str[256] = {}; + + if (*params) { + p = btf__type_by_id(btf, *params); + btf_loc_param_str(p, param_str, sizeof(param_str)); + } else { + snprintf(param_str, sizeof(param_str), ""); + } + if (json_output) { + jsonw_start_object(w); + jsonw_uint_field(w, "type_id", *params); + jsonw_string_field(w, "value", param_str); + jsonw_end_object(w); + } else { + printf("\n\ttype_id=%u value='%s'", *params, param_str); + } + } + if (json_output) + jsonw_end_array(w); + break; + } + + case BTF_KIND_LOCSEC: { + struct btf_loc *locs = btf_locsec_locs(t); + __u32 i, vlen = btf_vlen(t); + + if (json_output) { + jsonw_uint_field(w, "vlen", vlen); + jsonw_name(w, "locs"); + jsonw_start_array(w); + } else { + printf(" vlen=%u", vlen); + } + + for (i = 0; i < vlen; i++, locs++) { + const struct btf_type *f = btf__type_by_id(btf, locs->func); + const char *name = ""; + + if (f && btf_is_func(f)) + name = btf_str(btf, f->name_off); + + if (json_output) { + jsonw_start_object(w); + jsonw_uint_field(w, "func_type_id", locs->func); + jsonw_string_field(w, "name", name); + jsonw_uint_field(w, "loc_proto_type_id", locs->loc_proto); + jsonw_uint_field(w, "offset", locs->offset); + jsonw_end_object(w); + } else { + printf("\n\tname='%s' func_type_id=%u loc_proto_type_id=%u offset=%u", + name, locs->func, locs->loc_proto, locs->offset); + } + } + if (json_output) + jsonw_end_array(w); + break; + } default: break; } From 3429e01578b673fa4f7deb989b5c51d61a988326 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:27 +0100 Subject: [PATCH 259/660] selftests/bpf: Test bpftool dump of BTF location info Add a bpftool raw BTF dump test covering LOC_PARAM, LOC_PROTO, and LOCSEC types. Verify LOC_PARAM expressions, LOC_PROTO parameter values, and LOCSEC function name, type id, and offset output. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-11-alan.maguire@oracle.com --- .../bpf/prog_tests/bpftool_btf_dump.c | 121 +++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c index d5b25302b0c8..bed506badc75 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -57,6 +57,57 @@ static struct btf *mk_btf(void) return btf; } +static struct btf *mk_loc_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_func_proto(btf, 1); + btf__add_func_param(btf, "arg1", 1); + btf__add_func_param(btf, "arg2", 1); + btf__add_func_param(btf, "arg3", 1); + btf__add_func_param(btf, "arg4", 1); + btf__add_func_param(btf, "arg5", 1); + btf__add_func(btf, "foo", BTF_FUNC_STATIC, 2); + + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf, 1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET); + btf__add_loc_param_value(btf, 2); + btf__add_loc_param_value(btf, 0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0x9abcdef0); + btf__add_loc_param_value(btf, 0x12345678); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0xfeedface); + btf__add_loc_param_value(btf, 0xdeadbeef); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x20); + btf__add_loc_proto(btf); + btf__add_loc_proto_param(btf, 4); + btf__add_loc_proto_param(btf, 5); + btf__add_loc_proto_param(btf, 6); + btf__add_loc_proto_param(btf, 7); + btf__add_loc_proto_param(btf, 8); + btf__add_loc_proto_param(btf, 9); + btf__add_locsec(btf, "inline.text"); + btf__add_locsec_loc(btf, 3, 10, 64); + + return btf; +} + static int btf_to_tmpfile(const struct btf *btf, char *path) { ssize_t written; @@ -105,6 +156,26 @@ static char *dump_c(const char *btf_path, bool sorted) return buf; } +static char *dump_raw(const char *btf_path) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s", btf_path); + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_raw")) { + free(buf); + return NULL; + } + + return buf; +} + static char *read_expected(const char *path) { char *buf = NULL; @@ -155,10 +226,46 @@ out_dump: free(dump); } +static void test_loc_dump(const char *btf_path) +{ + const char expected[] = + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED\n" + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=5\n" + "\t'arg1' type_id=1\n" + "\t'arg2' type_id=1\n" + "\t'arg3' type_id=1\n" + "\t'arg4' type_id=1\n" + "\t'arg5' type_id=1\n" + "[3] FUNC 'foo' type_id=2 linkage=static\n" + "[4] LOC_PARAM '(anon)' size=4 flags=0x8 vlen=1 values='reg1'\n" + "[5] LOC_PARAM '(anon)' size=8 flags=0x38 vlen=2 values='*(reg2 + 0x10)'\n" + "[6] LOC_PARAM '(anon)' size=8 flags=0x29 vlen=2 values='fbreg - 0x10'\n" + "[7] LOC_PARAM '(anon)' size=8 flags=0x6 vlen=2 values='0x123456789abcdef0 (addr)'\n" + "[8] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2 values='0xdeadbeeffeedface'\n" + "[9] LOC_PARAM '(anon)' size=8 flags=0x39 vlen=2 values='*(fbreg - 0x20)'\n" + "[10] LOC_PROTO '(anon)' vlen=6\n" + "\ttype_id=4 value='reg1'\n" + "\ttype_id=5 value='*(reg2 + 0x10)'\n" + "\ttype_id=6 value='fbreg - 0x10'\n" + "\ttype_id=7 value='0x123456789abcdef0 (addr)'\n" + "\ttype_id=8 value='0xdeadbeeffeedface'\n" + "\ttype_id=9 value='*(fbreg - 0x20)'\n" + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tname='foo' func_type_id=3 loc_proto_type_id=10 offset=64\n"; + char *dump; + + dump = dump_raw(btf_path); + if (!dump) + return; + + ASSERT_OK(compare_text_to_expected(dump, expected), "cmp_loc_dump"); + free(dump); +} + void test_bpftool_btf_dump(void) { char path[PATH_MAX]; - struct btf *btf; + struct btf *btf = NULL; btf = mk_btf(); if (!btf) @@ -171,7 +278,19 @@ void test_bpftool_btf_dump(void) test_dump(path, true); if (test__start_subtest("c_unsorted")) test_dump(path, false); + unlink(path); + btf__free(btf); + + btf = mk_loc_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("loc_dump")) + test_loc_dump(path); unlink(path); out_btf: btf__free(btf); From ce2913d959a2355b868ad250ee9639252b4ab08d Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Thu, 24 Sep 2026 12:14:28 +0100 Subject: [PATCH 260/660] Documentation/bpf: Describe new location-related BTF kinds Update BTF specification to describe encoding schemes for BTF_KIND_LOC_PARAM, BTF_KIND_LOC_PROTO and BTF_KIND_LOCSEC. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924111428.75957-12-alan.maguire@oracle.com --- Documentation/bpf/btf.rst | 83 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 004aa1058d85..29de1222c3e7 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -88,6 +88,9 @@ sequentially and type id is assigned to each recognized type starting from id #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ #define BTF_KIND_ENUM64 19 /* Enumeration up to 64-bit values */ + #define BTF_KIND_LOC_PARAM 20 /* Location description (register, const etc) */ + #define BTF_KIND_LOC_PROTO 21 /* Set of location parameters for site */ + #define BTF_KIND_LOCSEC 22 /* Section with site descriptions */ Note that the type section encodes debug info, not just pure types. ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram. @@ -104,11 +107,13 @@ Each type contains the following common data:: * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64. + /* "size" is used by INT, ENUM, STRUCT, UNION, ENUM64 and + * LOC_PARAM. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. + * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. It is unused by + * LOC_PROTO and LOCSEC. * "type" is a type_id referring to another type. */ union { @@ -563,6 +568,80 @@ The ``btf_enum64`` encoding: If the original enum value is signed and the size is less than 8, that value will be sign extended into 8 bytes. +2.2.20 BTF_KIND_LOC_PARAM +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PARAM + * ``info.vlen``: number of 32-bit location value words + * ``size``: size in bytes of the represented parameter: 1 through 16 + +``btf_type`` is followed by a ``struct btf_loc_param`` and ``info.vlen`` +number of 32-bit value words.:: + + struct btf_loc_param { + __u32 flags; + __u32 values[]; + }; + +The ``flags`` field describes how to interpret ``values``: + + * ``BTF_LOC_PARAM_CONST`` describes a constant; the value is stored in + low-word, high-word order when it requires 64 bits. + * ``BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST`` describes an address offset + relative to the runtime base address of the kernel or module image. + * ``BTF_LOC_PARAM_REG`` with one word describes a register number; with two + words it describes a multi-register parameter. + * ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET`` describes an address held in + a register plus an offset. Adding ``BTF_LOC_PARAM_DEREF`` dereferences + that address. ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF`` with one word + dereferences the value held in the register. + * ``BTF_LOC_PARAM_SIGNED`` makes a constant or offset signed. A constant's + signed width is ``size``. For a register-relative offset, its signed + width is the number of offset value words times 32 bits. + +2.2.21 BTF_KIND_LOC_PROTO +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PROTO + * ``info.vlen``: number of function parameter locations + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``__u32`` BTF type IDs. +Each entry corresponds to a function parameter at an inline site. An entry is +either 0, meaning that no location information is available, or the type ID +of a ``BTF_KIND_LOC_PARAM``. + +2.2.22 BTF_KIND_LOCSEC +~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: offset to a valid ELF section name + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOCSEC + * ``info.vlen``: number of inline sites in the section + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``struct btf_loc``.:: + + struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; + }; + +The ``func`` field is the non-zero type ID of the ``BTF_KIND_FUNC`` being +described. ``loc_proto`` is the non-zero type ID of the associated +``BTF_KIND_LOC_PROTO``. ``offset`` is the location address offset relative to +the runtime base address of the ELF section associated with the LOCSEC. +For example, a LOCSEC named ``inline.text`` contains records for ``.text`` +whose offsets are relative to the runtime base of that section. + 2.3 Constant Values ------------------- From ec3a69e7c021cd826be8886ba6af4ee3147788fb Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 09:53:49 +0200 Subject: [PATCH 261/660] bpf: Check fastcall stack contract once for all stack accesses When the verifier inlines a bpf_fastcall helper or kfunc, it removes the spill/fill pairs that clang emitted around the call and lowers the subprogram's stack depth so that their slots are no longer part of the frame. This is only safe if nothing else accesses those slots or any slot below them. check_fastcall_stack_contract() enforces this: an access to that region from outside a fastcall pattern disables the rewrite for the subprogram. It is called from the four stack load and store paths, and two kinds of access escape it. Stack buffers passed to helpers and kfuncs are validated by check_stack_range_initialized(), which does not call it. If such a buffer is the only other access to the region, the rewrite is still applied, the JIT reserves a frame that does not contain the buffer, and the helper reads or writes live kernel stack below the program's frame. For example, without CAP_PERFMON: *(u64 *)(r10 - 8) = r1; call bpf_get_smp_processor_id; r1 = *(u64 *)(r10 - 8); r2 = 0; r3 = r10; r3 += -64; r4 = 8; call bpf_skb_load_bytes; is accepted with a stack depth of 0 instead of 64. Some buffers got the check indirectly. For constant-sized outputs in raw mode, mark_raw_stack() marks the buffer initialized through ordinary byte stores, which call the check. Commit 5da4a9f26fca ("bpf: Preserve stack initialization for generic output buffers") limited raw mode to programs that may read uninitialized stack, so outputs of programs without CAP_PERFMON lost the check, as in the example above. Buffers that never used raw mode have lacked the check since fastcall support was added: helper inputs such as a map key whose only store is a fastcall spill, variable-sized outputs, and buffers at a variable stack offset. check_stack_read_fixed_off() applies the check to the current frame, even when the pointer targets a caller's stack. A callee can therefore read a caller's fastcall spill slot without disabling the caller's rewrite: caller: r1 = 1; *(u64 *)(r10 - 8) = r1; call bpf_get_smp_processor_id; r1 = *(u64 *)(r10 - 8); r1 = r10; r1 += -8; call callee; callee: r0 = *(u64 *)(r1 + 0); The caller's spill and fill are removed and its stack depth drops to 0, but the verifier still tracks the slot as holding the spilled constant. The callee's load then reads kernel stack outside the caller's frame, while the verifier assumes r0 is 1. Every stack access first goes through check_stack_access_within_bounds(): loads and stores from check_mem_access(), helper and kfunc buffers from check_stack_range_initialized(), and the dynptr, iterator and irq flag slots that kfuncs initialize, whose handlers use check_mem_access(). It resolves the frame that owns the slots and the lowest offset the access can touch, and it is the only place where a frame's stack depth grows. Check the contract there, once for every access, and drop the calls from the stack read and write paths. Zero-sized buffers touch no stack and leave the rewrite enabled. Fixes: 5b5f51bff1b6 ("bpf: no_caller_saved_registers attribute for helper calls") Fixes: 5da4a9f26fca ("bpf: Preserve stack initialization for generic output buffers") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/85f9995a43edb70c76615280e103dc5325742e88330f36c97962ee35f17e3e32@mail.kernel.org Link: https://patch.msgid.link/20260924075352.2343553-2-memxor@gmail.com --- kernel/bpf/verifier.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d503c08ca3e7..4c20e5bf8b69 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3716,7 +3716,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (err) return err; - check_fastcall_stack_contract(env, state, insn_idx, off); mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { bool reg_value_fits; @@ -3837,7 +3836,6 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, return err; } - check_fastcall_stack_contract(env, state, insn_idx, min_off); /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { u8 new_type, *stype; @@ -4022,7 +4020,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, reg = ®_state->stack[spi].spilled_ptr; mark_stack_slot_scratched(env, spi); - check_fastcall_stack_contract(env, state, env->insn_idx, off); /* * Refine the in-progress load record's origin to the source stack slot. @@ -4209,7 +4206,6 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg dst_regno); if (err) return err; - check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -6614,6 +6610,15 @@ static int check_stack_access_within_bounds( return err; } + /* + * Every stack access passes through here, including buffers passed to + * helpers and kfuncs and accesses into a caller's frame, so check the + * fastcall contract of the frame that owns the slots once for all of + * them. Zero-sized accesses touch no stack and keep the rewrite enabled. + */ + if (access_size) + check_fastcall_stack_contract(env, state, env->insn_idx, min_off); + /* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ From ba28c18f3439c4d417fa9f5b2a69c973dd122339 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 09:53:50 +0200 Subject: [PATCH 262/660] selftests/bpf: Test fastcall rewrite with indirect stack accesses Add tests where a helper stack buffer, or a load through a pointer into another frame, overlaps the slots of a fastcall spill/fill pair. The rewrite must not be applied in these cases, so check that the spill and fill remain in the translated program and that the final stack depth still covers the accessed slots. Cover: - a constant-sized uninitialized output without CAP_PERFMON; - the same output in the caller's stack, passed to a helper by a callee; - a helper input whose only initialization is the fastcall spill; - a callee load from the caller's fastcall spill slot. Also add a zero-sized buffer, for which the rewrite must still be applied. The tests only load the programs and inspect the verifier log and the translated instructions. Do not run them: without the fixes, the verifier accepts programs that access memory outside their stack frame. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924075352.2343553-3-memxor@gmail.com --- .../bpf/progs/verifier_bpf_fastcall.c | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index a73b837553fb..e0c389102db9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -502,6 +502,175 @@ __naked void bad_helper_write(void) : __clobber_all); } +/* + * A helper buffer or a callee's pointer that reaches a fastcall spill slot + * must keep the spill/fill pair and the stack that covers it. Only load + * these programs: without the fix, they access kernel stack outside their + * frame. + * + * Uninitialized outputs without CAP_PERFMON have no explicit stack accesses. + */ +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = 0;" + "r3 = r10;" + "r3 += -64;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +/* Same output in the caller's stack, passed to the helper by a callee. */ +static __used __naked void helper_uninit_stack_callee(void) +{ + asm volatile ( + "r3 = r2;" + "r2 = 0;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "exit;" + : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack_caller) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack_caller(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = r10;" + "r2 += -64;" + "call helper_uninit_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A helper input whose only initialization is the fastcall spill. */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} fastcall_map SEC(".maps"); + +SEC("tc") +__log_level(4) +__msg("subprog 0 (helper_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void helper_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = %[fastcall_map] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_lookup_elem];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_map_lookup_elem), + __imm_addr(fastcall_map) + : __clobber_all); +} + +/* A callee load from the caller's fastcall spill slot. */ +static __used __naked void read_caller_stack_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("raw_tp") +__log_level(4) +__msg("subprog 0 (callee_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void callee_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call read_caller_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A zero-sized buffer touches no stack, the rewrite is still applied. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (helper_zero_size_buffer) main {{.*}} stack 0") +__xlated("0: r1 = 1") +__xlated("1: r0 =") +__success +__naked void helper_zero_size_buffer(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -64;" + "r2 = 0;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_probe_read_kernel) + : __clobber_all); +} + SEC("raw_tp") __arch_x86_64 /* main, not patched */ From 3b8da394c289f91242ea8088a65ed2d9a503c8cc Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:02 +0200 Subject: [PATCH 263/660] bpf: Add accessors for verifier stack slots The verifier indexes a frame's stack state directly through state->stack[spi] and computes the number of tracked slots as allocated_stack / BPF_REG_SIZE in every file that touches stack slots, and so does the nfp offload driver. Route all of these through two helpers, bpf_stack_slot() and bpf_stack_nr_slots(), so the layout of the per-frame stack state is visible in one place. The one lookup that goes the other way, reg_to_target() in the diagnostics code, maps a register pointer back to its slot by address and now says that it relies on the slots forming one contiguous array. Both helpers take a const frame: the slot accessor returns the slot through the frame's stack pointer, so read-only code such as the state printer can use it without giving up its qualifiers. Functions that look up the same slot repeatedly now fetch it once. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-2-memxor@gmail.com --- .../net/ethernet/netronome/nfp/bpf/verifier.c | 2 +- include/linux/bpf_verifier.h | 29 ++- kernel/bpf/backtrack.c | 18 +- kernel/bpf/diagnostics.c | 11 +- kernel/bpf/log.c | 13 +- kernel/bpf/states.c | 90 ++++----- kernel/bpf/verifier.c | 172 ++++++++++-------- 7 files changed, 193 insertions(+), 142 deletions(-) diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index 1caa87da72b5..9ddbfe9d6019 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -105,7 +105,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env) unsigned int soff; soff = -(off + i) - 1; - stack_entry = &state->stack[soff / BPF_REG_SIZE]; + stack_entry = bpf_stack_slot(state, soff / BPF_REG_SIZE); if (stack_entry->slot_type[soff % BPF_REG_SIZE] == STACK_ZERO) continue; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 73cb097859a1..6f47a3ccde86 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -530,12 +530,31 @@ struct bpf_verifier_state { u32 may_goto_depth; }; +/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */ +static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) +{ + return frame->allocated_stack / BPF_REG_SIZE; +} + +/* + * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see grow_stack_state(). + */ +static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) +{ + return &frame->stack[spi]; +} + static inline struct bpf_reg_state * bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) { - if (slot < frame->allocated_stack / BPF_REG_SIZE && - (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) - return &frame->stack[slot].spilled_ptr; + struct bpf_stack_state *ss; + + if (slot >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, slot); + if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask) + return &ss->spilled_ptr; return NULL; } @@ -551,7 +570,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \ - iter < frame->allocated_stack / BPF_REG_SIZE; \ + iter < bpf_stack_nr_slots(frame); \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ @@ -576,7 +595,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ - __stack = &__state->stack[___j]; \ + __stack = bpf_stack_slot(__state, ___j); \ (void)(__expr); \ } \ __stack = NULL; \ diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 4da99dec0818..9cc712f1619b 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -711,10 +711,12 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, i, j); } } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE || reg->precise) continue; reg->precise = true; @@ -826,6 +828,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, int subseq_idx = -1; struct bpf_func_state *func; bool tmp, skip_first = true; + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int i, fr, err; @@ -950,16 +953,17 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); for_each_set_bit(i, mask, 64) { - if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, + if (verifier_bug_if(i >= bpf_stack_nr_slots(func), env, "stack slot %d, total slots %d", - i, func->allocated_stack / BPF_REG_SIZE)) + i, bpf_stack_nr_slots(func))) return -EFAULT; - if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { + ss = bpf_stack_slot(func, i); + if (!bpf_is_spilled_scalar_reg(ss)) { bt_clear_frame_slot(bt, fr, i); continue; } - reg = &func->stack[i].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->precise) { bt_clear_frame_slot(bt, fr, i); } else { diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 5ecfa86ed49f..af97aea49075 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1600,9 +1600,9 @@ static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, return NULL; return &state->stack_arg_regs[target->stack_arg]; case BPF_DIAG_MOD_TARGET_STACK_SLOT: - if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + if (target->spi >= bpf_stack_nr_slots(state)) return NULL; - return &state->stack[target->spi].spilled_ptr; + return &bpf_stack_slot(state, target->spi)->spilled_ptr; default: return NULL; } @@ -1618,7 +1618,7 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta for (frame = 0; frame <= vstate->curframe; frame++) { struct bpf_func_state *state = vstate->frame[frame]; unsigned long start, end; - u32 nslots = state->allocated_stack / BPF_REG_SIZE; + u32 nslots = bpf_stack_nr_slots(state); int spi; start = (unsigned long)state->regs; @@ -1637,6 +1637,11 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta return true; } + /* + * Map the pointer back to its slot by address, which relies on + * the slots forming one contiguous array as bpf_stack_slot() + * indexes it. + */ start = (unsigned long)state->stack; end = (unsigned long)(state->stack + nslots); if (nslots && addr >= start && addr < end) { diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fb032dfdc0de..d850a7863d2e 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -716,7 +716,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, "="); print_reg_state(env, state, reg); } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *slot = bpf_stack_slot(state, i); char types_buf[BPF_REG_SIZE + 1]; const char *sep = ""; bool valid = false; @@ -727,7 +728,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie continue; for (j = 0; j < BPF_REG_SIZE; j++) { - slot_type = state->stack[i].slot_type[j]; + slot_type = slot->slot_type[j]; if (slot_type != STACK_INVALID && slot_type != STACK_POISON) valid = true; types_buf[j] = slot_type_char[slot_type]; @@ -736,12 +737,12 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie if (!valid) continue; - reg = &state->stack[i].spilled_ptr; - switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { + reg = &slot->spilled_ptr; + switch (slot->slot_type[BPF_REG_SIZE - 1]) { case STACK_SPILL: /* print MISC/ZERO/INVALID slots above subreg spill */ for (j = 0; j < BPF_REG_SIZE; j++) - if (state->stack[i].slot_type[j] == STACK_SPILL) + if (slot->slot_type[j] == STACK_SPILL) break; types_buf[j] = '\0'; @@ -751,7 +752,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie case STACK_DYNPTR: /* skip to main dynptr slot */ i += BPF_DYNPTR_NR_SLOTS - 1; - reg = &state->stack[i].spilled_ptr; + reg = &bpf_stack_slot(state, i)->spilled_ptr; verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 012b82513a3b..2df6903a9be9 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -415,14 +415,14 @@ static void __clean_func_state(struct bpf_verifier_env *env, * half_spi 2*i → lower half: slot_type[0..3] (closer to FP) * half_spi 2*i+1 → upper half: slot_type[4..7] (farther from FP) */ - for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(st); i++) { bool lo_live = bpf_stack_slot_alive(env, frame, i * 2); bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1); if (!hi_live || !lo_live) { int start = !lo_live ? 0 : BPF_REG_SIZE / 2; int end = !hi_live ? BPF_REG_SIZE : BPF_REG_SIZE / 2; - u8 stype = st->stack[i].slot_type[7]; + u8 stype = bpf_stack_slot(st, i)->slot_type[7]; /* * Don't clear special slots. @@ -442,7 +442,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, * rejecting as non-scalar register fills. */ if (!hi_live) { - struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; + struct bpf_reg_state *spill = &bpf_stack_slot(st, i)->spilled_ptr; if (lo_live && stype == STACK_SPILL) { if (spill->type != SCALAR_VALUE) @@ -454,7 +454,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (bpf_register_is_null(spill)) continue; for (j = 0; j < 4; j++) { - u8 *t = &st->stack[i].slot_type[j]; + u8 *t = &bpf_stack_slot(st, i)->slot_type[j]; if (*t == STACK_SPILL) *t = STACK_MISC; @@ -463,7 +463,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, spill); } for (j = start; j < end; j++) - st->stack[i].slot_type[j] = STACK_POISON; + bpf_stack_slot(st, i)->slot_type[j] = STACK_POISON; } } } @@ -709,37 +709,38 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * didn't use them */ for (i = 0; i < old->allocated_stack; i++) { + struct bpf_stack_state *old_slot, *cur_slot; struct bpf_reg_state *old_reg, *cur_reg; int im = i % BPF_REG_SIZE; + u8 old_type; spi = i / BPF_REG_SIZE; + old_slot = bpf_stack_slot(old, spi); + old_type = old_slot->slot_type[im]; + cur_slot = i < cur->allocated_stack ? bpf_stack_slot(cur, spi) : NULL; if (exact == EXACT) { - u8 old_type = old->stack[spi].slot_type[i % BPF_REG_SIZE]; - u8 cur_type = i < cur->allocated_stack ? - cur->stack[spi].slot_type[i % BPF_REG_SIZE] : STACK_INVALID; + u8 cur_type = cur_slot ? cur_slot->slot_type[im] : STACK_INVALID; /* STACK_INVALID and STACK_POISON are equivalent for pruning */ if (old_type == STACK_POISON) old_type = STACK_INVALID; if (cur_type == STACK_POISON) cur_type = STACK_INVALID; - if (i >= cur->allocated_stack || old_type != cur_type) + if (!cur_slot || old_type != cur_type) return false; } - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID || - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_POISON) + if (old_type == STACK_INVALID || old_type == STACK_POISON) continue; - if (env->allow_uninit_stack && - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) + if (env->allow_uninit_stack && old_type == STACK_MISC) continue; /* explored stack has more populated slots than current stack * and these slots were used */ - if (i >= cur->allocated_stack) + if (!cur_slot) return false; /* @@ -749,8 +750,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * regsafe() to ensure scalar ids are compared. */ if (im == 0 || im == 4) { - old_reg = scalar_reg_for_stack(env, &old->stack[spi], im); - cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im); + old_reg = scalar_reg_for_stack(env, old_slot, im); + cur_reg = scalar_reg_for_stack(env, cur_slot, im); if (old_reg && cur_reg) { if (!regsafe(env, old_reg, cur_reg, idmap, exact)) return false; @@ -763,21 +764,19 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * it will be safe with zero-initialized stack. * The opposite is not true */ - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && - cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) + if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO) continue; - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != - cur->stack[spi].slot_type[i % BPF_REG_SIZE]) + if (old_type != cur_slot->slot_type[im]) /* Ex: old explored (safe) state has STACK_SPILL in * this stack slot, but current has STACK_MISC -> * this verifier states are not equivalent, * return false to continue verification of this path */ return false; - if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) + if (im != BPF_REG_SIZE - 1) continue; /* Both old and cur are having same slot_type */ - switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { + switch (old_type) { case STACK_SPILL: /* when explored and current stack slot are both storing * spilled registers, check that stored pointers types @@ -789,13 +788,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * such verifier states are not equivalent. * return false to continue verification of this path */ - if (!regsafe(env, &old->stack[spi].spilled_ptr, - &cur->stack[spi].spilled_ptr, idmap, exact)) + if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr, + idmap, exact)) return false; break; case STACK_DYNPTR: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (old_reg->dynptr.type != cur_reg->dynptr.type || old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || !check_ids(old_reg->id, cur_reg->id, idmap) || @@ -803,8 +802,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_ITER: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; /* iter.depth is not compared between states as it * doesn't matter for correctness and would otherwise * prevent convergence; we maintain it only to prevent @@ -820,8 +819,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_IRQ_FLAG: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (!check_ids(old_reg->id, cur_reg->id, idmap) || old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class) return false; @@ -1045,10 +1044,12 @@ static int propagate_precision(struct bpf_verifier_env *env, first = false; } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (!bpf_is_spilled_reg(&state->stack[i])) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (!bpf_is_spilled_reg(ss)) continue; - state_reg = &state->stack[i].spilled_ptr; + state_reg = &ss->spilled_ptr; if (state_reg->type != SCALAR_VALUE || !state_reg->precise) continue; @@ -1194,15 +1195,17 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf for (fr = old->curframe; fr >= 0; fr--) { state = old->frame[fr]; - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].slot_type[0] != STACK_ITER) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (ss->slot_type[0] != STACK_ITER) continue; - slot = &state->stack[i].spilled_ptr; + slot = &ss->spilled_ptr; if (slot->iter.state != BPF_ITER_STATE_ACTIVE) continue; - cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; + cur_slot = &bpf_stack_slot(cur->frame[fr], i)->spilled_ptr; if (cur_slot->iter.depth != slot->iter.depth) return true; } @@ -1224,10 +1227,12 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ continue; reg->precise = false; } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE) continue; reg->precise = false; @@ -1330,7 +1335,7 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ if (is_iter_next_insn(env, insn_idx)) { if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { - struct bpf_func_state *cur_frame; + struct bpf_func_state *cur_frame, *iter_frame; struct bpf_reg_state *iter_state, *iter_reg; int spi; @@ -1344,7 +1349,8 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * no need for extra (re-)validations */ spi = bpf_get_spi(iter_reg->var_off.value); - iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr; + iter_frame = bpf_func(env, iter_reg); + iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr; if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { loop = true; goto hit; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4c20e5bf8b69..6b003e4ee450 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -598,16 +598,17 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn) static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) { - int allocated_slots = state->allocated_stack / BPF_REG_SIZE; + int allocated_slots = bpf_stack_nr_slots(state); - /* We need to check that slots between [spi - nr_slots + 1, spi] are - * within [0, allocated_stack). - * - * Please note that the spi grows downwards. For example, a dynptr - * takes the size of two stack slots; the first slot will be at - * spi and the second slot will be at spi - 1. - */ - return spi - nr_slots + 1 >= 0 && spi < allocated_slots; + /* + * We need to check that slots between [spi - nr_slots + 1, spi] are + * within [0, allocated_stack). + * + * Please note that the spi grows downwards. For example, a dynptr + * takes the size of two stack slots; the first slot will be at + * spi and the second slot will be at spi - 1. + */ + return spi - nr_slots + 1 >= 0 && spi < allocated_slots; } static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -751,8 +752,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return err; for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_DYNPTR; - state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR; } type = arg_to_dynptr_type(arg_type); @@ -785,8 +786,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ parent_id = dynptr->parent_id; } - mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, - &state->stack[spi - 1].spilled_ptr, type, parent_id); + mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr, + &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id); return 0; } @@ -818,7 +819,7 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re * all clones and derived slices. For non-referenced dynptr, only * the dynptr and slices derived from it will be invalidated. */ - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; return release_reference(env, dynptr_type_referenced(reg->dynptr.type) ? reg->parent_id : reg->id); @@ -857,6 +858,7 @@ static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int err = 0; /* We always ensure that STACK_DYNPTR is never set partially, @@ -864,20 +866,22 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, * different for STACK_SPILL, where it may be only set for * 1 byte, so code has to use is_spilled_reg. */ - if (state->stack[spi].slot_type[0] != STACK_DYNPTR) + if (slot->slot_type[0] != STACK_DYNPTR) return 0; /* Reposition spi to first slot */ - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + if (!slot->spilled_ptr.dynptr.first_slot) { spi = spi + 1; + slot = bpf_stack_slot(state, spi); + } /* * A referenced dynptr can be overwritten only if there is at * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ - if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && - dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { + if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) && + dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); bpf_diag_res( env, env->insn_idx, "referenced dynptr overwrite", @@ -887,7 +891,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, } /* Invalidate the dynptr and any derived slices */ - err = release_reference(env, state->stack[spi].spilled_ptr.id); + err = release_reference(env, slot->spilled_ptr.id); if (!err) { mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi - 1); @@ -927,6 +931,7 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct bpf_func_state *state = bpf_func(env, reg); + struct bpf_stack_state *slot; int i, spi; /* This already represents first slot of initialized bpf_dynptr. @@ -941,12 +946,13 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re spi = dynptr_get_spi(env, reg); if (spi < 0) return false; - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + slot = bpf_stack_slot(state, spi); + if (!slot->spilled_ptr.dynptr.first_slot) return false; for (i = 0; i < BPF_REG_SIZE; i++) { - if (state->stack[spi].slot_type[i] != STACK_DYNPTR || - state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) + if (slot->slot_type[i] != STACK_DYNPTR || + bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR) return false; } @@ -965,7 +971,7 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct if (spi < 0) return BPF_DYNPTR_TYPE_INVALID; state = bpf_func(env, reg); - return state->stack[spi].spilled_ptr.dynptr.type; + return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; } static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -1005,7 +1011,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, return id; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -1042,7 +1048,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, return spi; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (i == 0) @@ -1076,7 +1082,7 @@ static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, return false; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] == STACK_ITER) @@ -1097,7 +1103,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s return -EINVAL; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (st->type & PTR_UNTRUSTED) @@ -1139,7 +1145,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, if (id < 0) return id; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -1166,7 +1172,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return spi; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (st->irq.kfunc_class != kfunc_class) { @@ -1235,7 +1241,7 @@ static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bp if (spi < 0) return false; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] == STACK_IRQ_FLAG) @@ -1254,7 +1260,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return -EINVAL; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (!st->id) @@ -1401,7 +1407,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { - size_t n = src->allocated_stack / BPF_REG_SIZE; + size_t n = bpf_stack_nr_slots(src); dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), GFP_KERNEL_ACCOUNT); @@ -1440,7 +1446,7 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n) */ static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { - size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; + size_t old_n = bpf_stack_nr_slots(state), n; /* The stack size is always a multiple of BPF_REG_SIZE. */ size = round_up(size, BPF_REG_SIZE); @@ -3594,17 +3600,18 @@ static void save_register_state(struct bpf_verifier_env *env, int spi, struct bpf_reg_state *reg, int size) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; - bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); - state->stack[spi].spilled_ptr = *reg; + bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL); + slot->spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) - state->stack[spi].slot_type[i - 1] = STACK_SPILL; + slot->slot_type[i - 1] = STACK_SPILL; /* size < 8 bytes spill */ for (; i; i--) - mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + mark_stack_slot_misc(env, &slot->slot_type[i - 1]); bpf_diag_mod_end(env); } @@ -3646,14 +3653,15 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, static void scrub_special_slot(struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; /* regular write of data into stack destroys any spilled ptr */ - state->stack[spi].spilled_ptr.type = NOT_INIT; + slot->spilled_ptr.type = NOT_INIT; /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ - if (is_stack_slot_special(&state->stack[spi])) + if (is_stack_slot_special(slot)) for (i = 0; i < BPF_REG_SIZE; i++) - scrub_spilled_slot(&state->stack[spi].slot_type[i]); + scrub_spilled_slot(&slot->slot_type[i]); } /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, @@ -3671,13 +3679,14 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg = NULL; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(state, spi); /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && - bpf_is_spilled_reg(&state->stack[spi]) && - !bpf_is_spilled_scalar_reg(&state->stack[spi]) && + bpf_is_spilled_reg(ss) && + !bpf_is_spilled_scalar_reg(ss) && size != BPF_REG_SIZE) { const char *reason; @@ -3699,7 +3708,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[(slot - i) % + u8 type = ss->slot_type[(slot - i) % BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { @@ -3727,7 +3736,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - clear_scalar_id(&state->stack[spi].spilled_ptr); + clear_scalar_id(&ss->spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -3751,8 +3760,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; - if (bpf_is_spilled_reg(&state->stack[spi])) - bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + if (bpf_is_spilled_reg(ss)) + bpf_diag_record_scrub(env, &ss->spilled_ptr, BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); @@ -3773,7 +3782,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + ss->slot_type[(slot - i) % BPF_REG_SIZE] = type; insn_flags = 0; /* not a register spill */ } @@ -3838,12 +3847,14 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { + struct bpf_stack_state *ss; u8 new_type, *stype; int slot, spi; slot = -i - 1; spi = slot / BPF_REG_SIZE; - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; mark_stack_slot_scratched(env, spi); if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { @@ -3866,9 +3877,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, /* If writing_zero and the spi slot contains a spill of value 0, * maintain the spill type. */ - if (writing_zero && *stype == STACK_SPILL && - bpf_is_spilled_scalar_reg(&state->stack[spi])) { - struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; + if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) { + struct bpf_reg_state *spill_reg = &ss->spilled_ptr; if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { zero_used = true; @@ -3948,13 +3958,13 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, slot = -i - 1; spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); - stype = ptr_state->stack[spi].slot_type; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { zeros++; continue; } if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && - bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { + bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { zero_spill_mask |= 1ull << spi; zeros++; continue; @@ -4015,9 +4025,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi); - stype = reg_state->stack[spi].slot_type; - reg = ®_state->stack[spi].spilled_ptr; + stype = ss->slot_type; + reg = &ss->spilled_ptr; mark_stack_slot_scratched(env, spi); @@ -4027,7 +4038,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno >= 0) bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); - if (bpf_is_spilled_reg(®_state->stack[spi])) { + if (bpf_is_spilled_reg(ss)) { u8 spill_size = 1; for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) @@ -7428,6 +7439,7 @@ static int check_stack_range_initialized( } for (i = min_off; i < max_off + access_size; i++) { + struct bpf_stack_state *ss; u8 *stype; slot = -i - 1; @@ -7437,7 +7449,8 @@ static int check_stack_range_initialized( return -EFAULT; } - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || @@ -7449,13 +7462,13 @@ static int check_stack_range_initialized( goto mark; } - if (bpf_is_spilled_reg(&state->stack[spi]) && - (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || + if (bpf_is_spilled_reg(ss) && + (ss->spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { - __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); + __mark_reg_unknown(env, &ss->spilled_ptr); for (j = 0; j < BPF_REG_SIZE; j++) - scrub_spilled_slot(&state->stack[spi].slot_type[j]); + scrub_spilled_slot(&ss->slot_type[j]); } goto mark; } @@ -8185,7 +8198,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; } meta->dynptr.type = reg->dynptr.type; @@ -8318,7 +8331,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr); + update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -8395,16 +8408,15 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, &fold->regs[i], &fcur->regs[i]); - num_slots = min(fold->allocated_stack / BPF_REG_SIZE, - fcur->allocated_stack / BPF_REG_SIZE); + num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur)); for (i = 0; i < num_slots; i++) { - if (!bpf_is_spilled_reg(&fold->stack[i]) || - !bpf_is_spilled_reg(&fcur->stack[i])) + struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i); + struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i); + + if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss)) continue; - maybe_widen_reg(env, - &fold->stack[i].spilled_ptr, - &fcur->stack[i].spilled_ptr); + maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr); } } return 0; @@ -8416,7 +8428,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; - return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; + return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr; } /* process_iter_next_call() is called when verifier gets to iterator's next @@ -9208,6 +9220,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, s64 *value) { struct bpf_func_state *state = bpf_func(env, key); + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int slot, spi, off; int spill_size = 0; @@ -9227,7 +9240,8 @@ static int get_constant_map_key(struct bpf_verifier_env *env, slot = -stack_off - 1; spi = slot / BPF_REG_SIZE; off = slot % BPF_REG_SIZE; - stype = state->stack[spi].slot_type; + ss = bpf_stack_slot(state, spi); + stype = ss->slot_type; /* First handle precisely tracked STACK_ZERO */ for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) @@ -9238,14 +9252,14 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } /* Check that stack contains a scalar spill of expected size */ - if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) + if (!bpf_is_spilled_scalar_reg(ss)) return -EOPNOTSUPP; for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) spill_size++; if (spill_size != key_size) return -EOPNOTSUPP; - reg = &state->stack[spi].spilled_ptr; + reg = &ss->spilled_ptr; if (!tnum_is_const(reg->var_off)) /* Stack value not statically known */ return -EOPNOTSUPP; @@ -17841,10 +17855,12 @@ static void collect_linked_regs(struct bpf_verifier_env *env, reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; __collect_linked_regs(linked_regs, reg, id, i, j, false); } } @@ -17864,7 +17880,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s for (i = 0; i < linked_regs->cnt; ++i) { e = &linked_regs->entries[i]; reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] - : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr; + : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; if (reg->type != SCALAR_VALUE || reg == known_reg) continue; if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) From a9082d8a54c6b68ea4bc3417eda5b22138ac06c0 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:03 +0200 Subject: [PATCH 264/660] bpf: Widen the stack slot index in the jump history Jump history entries record the stack slot touched by a spill or fill in a 6-bit field, which only fits the 64 slots of a 512-byte frame and is pinned to that size by a static_assert on MAX_BPF_STACK. Move the flags into the first word and give the slot index 12 bits of the second word instead, so the entry stays 16 bytes while frames of up to 32 KiB can be recorded. Introduce MAX_BPF_STACK_SLOTS for the number of 8-byte slots a frame can have and use it for the static_assert and for BPF_ID_MAP_SIZE, so the verifier expresses per-frame capacity through one constant. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-3-memxor@gmail.com --- include/linux/bpf_verifier.h | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6f47a3ccde86..b1e88016edb1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -243,6 +243,14 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* + * Largest number of BPF_REG_SIZE stack slots a single frame can have. A frame + * may use any part of the MAX_BPF_STACK budget; check_max_stack_depth() + * enforces the bound on the combined depth of frames sharing the kernel stack + * and on each frame using a private stack. + */ +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK / BPF_REG_SIZE) + /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 @@ -425,12 +433,11 @@ struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; u32 frame : 4; /* stack access frame number */ - u32 spi : 6; /* stack slot index (0..63) */ - u32 : 2; - u32 prev_idx : 20; /* special INSN_F_xxx flags */ u32 flags : 4; - u32 : 8; + u32 : 4; + u32 prev_idx : 20; + u32 spi : 12; /* stack slot index */ /* * additional registers that need precision tracking when this * jump is backtracked, vector of five 11-bit records @@ -439,12 +446,12 @@ struct bpf_jmp_history_entry { }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); -static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); /* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ - MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; From 9eda94d7e89894024e5178d91e140b6fb0835d8e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:04 +0200 Subject: [PATCH 265/660] bpf: Store linked registers in the jump history as an array Linked scalar registers are recorded in the jump history packed into a u64 as five 11-bit entries, each naming a frame and a register or stack slot. The 6-bit slot field covers exactly the 64 slots of a 512-byte frame, so a spilled scalar in a deeper slot could not be linked and larger frames were ruled out by construction. Store the linked registers as an array of five u16 entries plus a count instead, each entry holding the frame number, a register-or-slot bit and an 11-bit register or slot index, which covers frames of up to 16 KiB. Callers that record no linked registers pass NULL. The history entry grows from 16 to 20 bytes, and the history is the one verifier structure whose size follows the number of instructions a loop iterates over rather than the state count. Measured over the 5075 selftest programs, peak verifier memory is unchanged for all but the loop-heavy ones, which grow by 8 to 16%: loop1/nested_loops from 17.6 to 19.2 MiB, verifier_loops1/jumps_out_rather_than_in from 4.4 to 5.1 MiB, strobemeta by 0.3% and pyperf600_nounroll by 0.8%. Verdicts, processed instructions and state counts stay the same everywhere. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-4-memxor@gmail.com --- include/linux/bpf_verifier.h | 16 +++++-- kernel/bpf/backtrack.c | 18 +++++--- kernel/bpf/states.c | 2 +- kernel/bpf/verifier.c | 86 ++++++++++++++++-------------------- 4 files changed, 62 insertions(+), 60 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b1e88016edb1..7d5da5b38e0e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -429,6 +429,9 @@ enum { INSN_F_STACK_ARG_ACCESS = BIT(3), }; +/* Registers linked to one jump condition that a history entry can record */ +#define BPF_LINKED_REGS_MAX 5 + struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; @@ -439,10 +442,14 @@ struct bpf_jmp_history_entry { u32 prev_idx : 20; u32 spi : 12; /* stack slot index */ /* - * additional registers that need precision tracking when this - * jump is backtracked, vector of five 11-bit records + * Scalar registers and spilled scalars linked to the condition of + * this jump, which need precision tracking together when the jump is + * backtracked. Each is packed as 4 bits of frame number, one bit + * telling a register from a stack slot and 11 bits of register or + * slot index, see linked_regs_pack(). */ - u64 linked_regs; + u16 linked_regs[BPF_LINKED_REGS_MAX]; + u8 linked_regs_cnt; }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); @@ -1292,7 +1299,8 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs); + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt); void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 9cc712f1619b..06630acd0d1f 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -9,7 +9,8 @@ /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs) + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; @@ -27,10 +28,13 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; - verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, - "insn history: insn_idx %d linked_regs: %#llx", - env->insn_idx, env->cur_hist_ent->linked_regs); - env->cur_hist_ent->linked_regs = linked_regs; + verifier_bug_if(env->cur_hist_ent->linked_regs_cnt != 0, env, + "insn history: insn_idx %d has %u linked regs", + env->insn_idx, env->cur_hist_ent->linked_regs_cnt); + if (linked_regs_cnt) + memcpy(env->cur_hist_ent->linked_regs, linked_regs, + linked_regs_cnt * sizeof(*linked_regs)); + env->cur_hist_ent->linked_regs_cnt = linked_regs_cnt; return 0; } @@ -47,7 +51,9 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state p->flags = insn_flags; p->spi = spi; p->frame = frame; - p->linked_regs = linked_regs; + if (linked_regs_cnt) + memcpy(p->linked_regs, linked_regs, linked_regs_cnt * sizeof(*linked_regs)); + p->linked_regs_cnt = linked_regs_cnt; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 2df6903a9be9..5c7169bfb4e9 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1418,7 +1418,7 @@ hit: */ err = 0; if (bpf_is_jmp_point(env, env->insn_idx)) - err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, cur, 0, 0, 0, NULL, 0); err = err ? : propagate_precision(env, &sl->state, cur, NULL); if (err) return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6b003e4ee450..401540bf6586 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3379,26 +3379,25 @@ static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].non_stack_access = true; } +/* Layout of one packed linked register in the jump history, see linked_regs_pack() */ #define LR_FRAMENO_BITS 4 -#define LR_SPI_BITS 6 -#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) -#define LR_SIZE_BITS 4 -#define LR_FRAMENO_MASK ((1ull << LR_FRAMENO_BITS) - 1) -#define LR_SPI_MASK ((1ull << LR_SPI_BITS) - 1) -#define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1) -#define LR_SPI_OFF LR_FRAMENO_BITS -#define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS) -#define LINKED_REGS_MAX 5 +#define LR_INDEX_BITS 11 +#define LR_FRAMENO_MASK ((1u << LR_FRAMENO_BITS) - 1) +#define LR_IS_REG BIT(LR_FRAMENO_BITS) +#define LR_INDEX_OFF (LR_FRAMENO_BITS + 1) +#define LR_INDEX_MASK ((1u << LR_INDEX_BITS) - 1) +#define LINKED_REGS_MAX BPF_LINKED_REGS_MAX static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); -static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS)); -static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64); +static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS)); +static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16); struct linked_reg { u8 frameno; union { - u8 spi; - u8 regno; + u16 spi; + u16 regno; }; bool is_reg; }; @@ -3417,48 +3416,34 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s) } /* - * Use u64 as a vector of 5 11-bit values, use first 4-bits to track - * number of elements currently in stack. - * Pack one history entry for linked registers as 11 bits in the following format: - * - 4-bits frameno - * - 6-bits spi_or_reg - * - 1-bit is_reg + * Pack linked registers for a jump history entry, one u16 each: + * - 4 bits frameno + * - 1 bit is_reg + * - 11 bits register or stack slot index */ -static u64 linked_regs_pack(struct linked_regs *s) +static void linked_regs_pack(const struct linked_regs *s, u16 *packed) { - u64 val = 0; int i; for (i = 0; i < s->cnt; ++i) { - struct linked_reg *e = &s->entries[i]; - u64 tmp = 0; + const struct linked_reg *e = &s->entries[i]; - tmp |= e->frameno; - tmp |= e->spi << LR_SPI_OFF; - tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF; - - val <<= LR_ENTRY_BITS; - val |= tmp; + packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF); } - val <<= LR_SIZE_BITS; - val |= s->cnt; - return val; } -static void linked_regs_unpack(u64 val, struct linked_regs *s) +static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s) { int i; - s->cnt = val & LR_SIZE_MASK; - val >>= LR_SIZE_BITS; - + s->cnt = hist->linked_regs_cnt; for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i]; + u16 packed = hist->linked_regs[i]; - e->frameno = val & LR_FRAMENO_MASK; - e->spi = (val >> LR_SPI_OFF) & LR_SPI_MASK; - e->is_reg = (val >> LR_IS_REG_OFF) & 0x1; - val >>= LR_ENTRY_BITS; + e->frameno = packed & LR_FRAMENO_MASK; + e->is_reg = packed & LR_IS_REG; + e->spi = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK; } } @@ -3500,10 +3485,10 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_ bool some_precise = false; int i; - if (!hist || hist->linked_regs == 0) + if (!hist || !hist->linked_regs_cnt) return; - linked_regs_unpack(hist->linked_regs, &linked_regs); + linked_regs_unpack(hist, &linked_regs); for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i]; @@ -3788,7 +3773,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -4164,7 +4149,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -4353,7 +4338,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } /* @@ -4387,7 +4372,7 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st cur->regs[dst_regno] = *arg; bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) @@ -17998,7 +17983,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } if (insn_flags) { - err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); + err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0); if (err) return err; } @@ -18068,7 +18053,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (linked_regs.cnt > 1) { - err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); + u16 packed[LINKED_REGS_MAX]; + + linked_regs_pack(&linked_regs, packed); + err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt); if (err) return err; } @@ -19466,7 +19454,7 @@ static int do_check(struct bpf_verifier_env *env) } if (bpf_is_jmp_point(env, env->insn_idx)) { - err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0); if (err) return err; } From 765e7328d64dec47a33fd7ac8c070b411297c275 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:05 +0200 Subject: [PATCH 266/660] bpf: Track backtracking stack slots with bitmaps Precision backtracking keeps the stack slots that still need a precise mark in one u64 per frame, which ties it to frames of at most 64 slots. Turn the per-frame masks into bitmaps sized by MAX_BPF_STACK_SLOTS and use the bitmap helpers for setting, clearing, testing and iterating them. The formatting helper takes a bitmap and the leftover-slot bug reports print the formatted slot list instead of a hex mask. mark_reg_stack_read() collected zero spills in a u64 of its own before handing it to the backtracker; it now counts them and revisits the range to mark each slot, which drops the only remaining mask-typed entry point. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-5-memxor@gmail.com --- include/linux/bpf_verifier.h | 13 +++----- kernel/bpf/backtrack.c | 65 ++++++++++++++++++++++-------------- kernel/bpf/verifier.c | 15 ++++++--- 3 files changed, 54 insertions(+), 39 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7d5da5b38e0e..781dc5ca1fe2 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -900,7 +900,7 @@ struct backtrack_state { struct bpf_verifier_env *env; u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; - u64 stack_masks[MAX_CALL_FRAMES]; + unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)]; u8 stack_arg_masks[MAX_CALL_FRAMES]; }; @@ -1367,12 +1367,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] |= 1ull << slot; -} - -static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) -{ - bt->stack_masks[frame] |= mask; + __set_bit(slot, bt->stack_masks[frame]); } static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) @@ -1387,7 +1382,7 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[frame] & (1ull << slot); + return test_bit(slot, bt->stack_masks[frame]); } bool bpf_map_is_rdonly(const struct bpf_map *map); @@ -1588,7 +1583,7 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); /* Kinds of member a by-value struct or union may be composed of. */ diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 06630acd0d1f..0e38b9575328 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -129,13 +129,26 @@ static inline void bt_reset(struct backtrack_state *bt) bt->env = env; } -static inline u32 bt_empty(struct backtrack_state *bt) +static inline bool bt_frame_stack_empty(struct backtrack_state *bt, u32 frame) { - u64 mask = 0; + return bitmap_empty(bt->stack_masks[frame], MAX_BPF_STACK_SLOTS); +} + +static inline bool bt_stack_empty(struct backtrack_state *bt) +{ + return bt_frame_stack_empty(bt, bt->frame); +} + +static inline bool bt_empty(struct backtrack_state *bt) +{ + u32 mask = 0; int i; - for (i = 0; i <= bt->frame; i++) - mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; + for (i = 0; i <= bt->frame; i++) { + mask |= bt->reg_masks[i] | bt->stack_arg_masks[i]; + if (!bt_frame_stack_empty(bt, i)) + return false; + } return mask == 0; } @@ -187,7 +200,7 @@ static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] &= ~(1ull << slot); + __clear_bit(slot, bt->stack_masks[frame]); } static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) @@ -200,14 +213,14 @@ static inline u32 bt_reg_mask(struct backtrack_state *bt) return bt->reg_masks[bt->frame]; } -static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) +static inline unsigned long *bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) { return bt->stack_masks[frame]; } -static inline u64 bt_stack_mask(struct backtrack_state *bt) +static inline unsigned long *bt_stack_mask(struct backtrack_state *bt) { - return bt->stack_masks[bt->frame]; + return bt_frame_stack_mask(bt, bt->frame); } static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) @@ -239,17 +252,16 @@ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) break; } } -/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) + +/* format stack slots bitmask, e.g., "-8,-24,-40" for slots 0, 2 and 4 */ +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask) { - DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; - bitmap_from_u64(mask, stack_mask); - for_each_set_bit(i, mask, 64) { + for_each_set_bit(i, stack_mask, MAX_BPF_STACK_SLOTS) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; @@ -461,10 +473,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, - "static subprog leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "static subprog leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* propagate r1-r5 to the caller */ @@ -497,9 +510,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, "callback leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "callback leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ @@ -881,7 +896,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, if (st->curframe == 0 && st->frame[0]->subprogno > 0 && st->frame[0]->callsite == BPF_MAIN_FUNC && - bt_stack_mask(bt) == 0 && + bt_stack_empty(bt) && (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { bitmap_from_u64(mask, bt_reg_mask(bt)); for_each_set_bit(i, mask, 32) { @@ -895,8 +910,9 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return 0; } - verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", - st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); + verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %s", + st->frame[0]->subprogno, bt_reg_mask(bt), env->tmp_str_buf); return -EFAULT; } @@ -957,8 +973,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } } - bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); - for_each_set_bit(i, mask, 64) { + for_each_set_bit(i, bt_frame_stack_mask(bt, fr), MAX_BPF_STACK_SLOTS) { if (verifier_bug_if(i >= bpf_stack_nr_slots(func), env, "stack slot %d, total slots %d", i, bpf_stack_nr_slots(func))) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 401540bf6586..2b87cb9beea0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3934,10 +3934,9 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; - u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; - int zeros = 0; + int zeros = 0, zero_spills = 0; for (i = min_off; i < max_off; i++) { slot = -i - 1; @@ -3950,7 +3949,7 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, } if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { - zero_spill_mask |= 1ull << spi; + zero_spills++; zeros++; continue; } @@ -3961,8 +3960,14 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); - if (zero_spill_mask) { - bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + if (zero_spills) { + for (i = min_off; i < max_off; i++) { + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL) + bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi); + } return mark_chain_precision_batch(env, env->cur_state); } } else { From ec373f1f0788a498a56ea73884bfe6ce23969d7e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:06 +0200 Subject: [PATCH 267/660] bpf: Track scratched stack slots with a bitmap The set of stack slots touched since the verifier state was last printed lives in a u64, so the log could only ever mark the first 64 slots of a frame as scratched. Make it a bitmap sized by MAX_BPF_STACK_SLOTS and go through the bitmap helpers for setting, testing, clearing and filling it. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-6-memxor@gmail.com --- include/linux/bpf_verifier.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 781dc5ca1fe2..e64cfb3fe675 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1092,7 +1092,7 @@ struct bpf_verifier_env { */ u32 scratched_regs; /* Same as scratched_regs but for stack slots */ - u64 scratched_stack_slots; + DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS); u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ struct bpf_reg_state fake_reg[1]; @@ -1520,7 +1520,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) { - env->scratched_stack_slots |= 1ULL << spi; + __set_bit(spi, env->scratched_stack_slots); } static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) @@ -1528,27 +1528,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) return (env->scratched_regs >> regno) & 1; } -static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi) { - return (env->scratched_stack_slots >> regno) & 1; + return test_bit(spi, env->scratched_stack_slots); } static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) { - return env->scratched_regs || env->scratched_stack_slots; + return env->scratched_regs || + !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) { env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; + bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } /* Used for printing the entire verifier state. */ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) { env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; + bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) From 1e054ab4aad667c937876d4b761ed5aa1f1a017a Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:07 +0200 Subject: [PATCH 268/660] bpf: Treat unknown-size stack reads as reaching the frame top When the size of a helper or kfunc memory argument is not a constant on the path, the stack liveness analysis is told the call reads MAX_BPF_STACK bytes starting at the pointer's offset. Clipped at the top of the frame this covers everything from the offset upwards, which is the intent, but only as long as no frame is deeper than MAX_BPF_STACK bytes. Return the "unknown" marker instead, which the liveness analysis already turns into a read of every slot between the offset and the frame top, independent of how deep the frame is. No functional change. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-7-memxor@gmail.com --- kernel/bpf/verifier.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2b87cb9beea0..a7b1142b0d51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14385,11 +14385,11 @@ scan_all_maps: } /* * Size arg is const on each path but differs across merged - * paths. MAX_BPF_STACK is a safe upper bound for reads. + * paths. Reads may extend anywhere up to the frame top. */ if (full_write) return 0; - return MAX_BPF_STACK; + return S64_MIN; } return S64_MIN; case ARG_PTR_TO_DYNPTR: @@ -14475,7 +14475,8 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * size = (s64)aux->const_reg_vals[size_reg]; goto out; } - return MAX_BPF_STACK; + /* Unknown size: the read may extend anywhere up to the frame top. */ + return S64_MIN; } /* fixed-size pointed-to type: resolve via BTF */ From 481ceda77aeb180022a3ab1e17a9929bcad353c9 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:08 +0200 Subject: [PATCH 269/660] bpf: Size liveness stack masks by the stack each frame uses Stack liveness tracks three masks of 4-byte stack slots - may_read, must_write and live_before - for every instruction of every frame of every function instance. Each mask used to be a fixed 128-bit spis_t, wide enough for the deepest frame MAX_BPF_STACK allows, so an instruction paid 48 bytes per frame no matter how little stack the frame actually touched. These per instruction arrays are the largest liveness allocation: a 40k instruction program spends ~2 MiB on each frame array. Sizing them for a larger stack budget would multiply that cost for every program, while most frames use a fraction of the budget. Replace the fixed masks with a variable width bitmap array per (instance, frame). struct frame_masks holds the width shared by all of its masks in @words plus a flat unsigned long array, where mask @kind of the instruction at relative index @i lives at &bits[(i * FM_MASK_CNT + kind) * words]. An array starts at the width the first recorded half-slot needs (minimum one word, that is 256 bytes of stack with 64-bit words) and is reallocated into a wider stride once a deeper half-slot shows up. Only the marking functions widen an array, and they all run during the static analysis before do_check(), so no verification time path can move one. The marking API now takes an inclusive half-slot range, which is what record_stack_access_off() computed anyway, and clamps the upper end to the deepest half-slot a frame can have. An access below the stack bound is rejected by the main verifier pass later; liveness only has to stay inside the masks. A range that covers no whole half-slot, as a byte store at fp-1 produces, marks nothing. A "read everything" mark, used for calls that are neither helpers nor kfuncs and for pointers whose offset or frame identity was lost, widens the array to the maximum width and sets every bit: recorded at a narrower width it would lose the half-slots a later widening adds, and no write can cancel it as no write covers the whole frame. A half-slot past the width of the masks was never read by the frame and is never live. merge_instances() may see a different width on each side, so it widens dst to cover src and counts a word only dst has as zero on the src side, which unions into may_read as a no-op and intersects must_write to empty, never claiming a write that did not happen. The rest is mechanical: update_insn() propagates live_before word by word over the array's width (all instructions of one array share it, hence so do an instruction's successors), is_live_before() answers false past the width, and the use:/def: log printing iterates up to the width. Liveness results are unchanged. A half-slot only ever gets a bit from a recorded access and recording one widens the array to cover it, so every bit the fixed masks could hold still fits. Frames whose deepest recorded half-slot fits in a single word now cost 24 bytes per instruction per frame instead of 48. The spill tracker that feeds the analysis had the same limit built in: it followed 64 slots, fp-8 to fp-512, per subprog, and a fill from a deeper slot returned an imprecise pointer, so every later access through it counted as a read of the whole stack of every frame. Let it follow the deepest 8-byte access a subprog makes directly through R10 when that reaches past those 64 slots, which is how spills and fills are compiled, and keep the count with the per-callsite snapshot a callee uses to fill from its caller's frame. Programs within 512 bytes are tracked exactly as before; a program spilling deeper stays precise. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-8-memxor@gmail.com --- include/linux/bpf_verifier.h | 50 +--- kernel/bpf/liveness.c | 507 +++++++++++++++++++++++------------ 2 files changed, 344 insertions(+), 213 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e64cfb3fe675..d3c31105a3ac 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -254,52 +254,6 @@ enum bpf_stack_slot_type { /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 -#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ - -typedef struct { - u64 v[2]; -} spis_t; - -#define SPIS_ZERO ((spis_t){}) -#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) - -static inline bool spis_is_zero(spis_t s) -{ - return s.v[0] == 0 && s.v[1] == 0; -} - -static inline bool spis_equal(spis_t a, spis_t b) -{ - return a.v[0] == b.v[0] && a.v[1] == b.v[1]; -} - -static inline spis_t spis_or(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; -} - -static inline spis_t spis_and(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; -} - -static inline spis_t spis_not(spis_t s) -{ - return (spis_t){{ ~s.v[0], ~s.v[1] }}; -} - -static inline bool spis_test_bit(spis_t s, u32 slot) -{ - return s.v[slot / 64] & BIT_ULL(slot % 64); -} - -static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) -{ - u32 w; - - for (w = lo; w <= hi && w < STACK_SLOTS; w++) - mask->v[w / 64] |= BIT_ULL(w % 64); -} #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ @@ -1047,8 +1001,8 @@ struct bpf_verifier_env { } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; - /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ - struct arg_track **callsite_at_stack; + /* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */ + struct spill_snapshot **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index a204b492acc8..1d83a4cf6ec5 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -10,10 +10,39 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) -struct per_frame_masks { - spis_t may_read; /* stack slots that may be read by this instruction */ - spis_t must_write; /* stack slots written by this instruction */ - spis_t live_before; /* stack slots that may be read by this insn and its successors */ +/* + * Stack liveness is tracked with a 4-byte (half register) granularity. + * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on, + * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have. + */ +#define FRAME_HALF_SPIS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) +#define FRAME_MAX_WORDS BITS_TO_LONGS(FRAME_HALF_SPIS) + +/* Masks tracked for each instruction of a frame */ +enum { + FM_MAY_READ, /* stack slots that may be read by this instruction */ + FM_MUST_WRITE, /* stack slots written by this instruction */ + FM_LIVE_BEFORE, /* stack slots that may be read by this insn and its successors */ + FM_MASK_CNT, +}; + +/* + * Per instruction stack masks for one frame of a function instance. + * + * Most frames use only a fraction of the stack budget, so instead of masks + * wide enough for every half-slot of the largest possible frame, all masks + * of one array share the width @words, and the marking functions widen the + * array as deeper half-slots are recorded. Mask @kind of the instruction at + * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A + * half-slot at or past @words * BITS_PER_LONG is never read by this frame, + * hence never live. An instruction that may read the whole frame, such as a + * call passing a frame pointer to another subprog, widens the array to + * FRAME_MAX_WORDS, so that the read cannot lose half-slots to a later + * widening. + */ +struct frame_masks { + u32 words; + unsigned long bits[]; }; /* @@ -29,7 +58,7 @@ struct func_instance { u32 subprog_start; /* cached env->subprog_info[subprog].start */ u32 insn_cnt; /* cached number of insns in the function */ /* Per frame, per instruction masks, frames allocated lazily. */ - struct per_frame_masks *frames[MAX_CALL_FRAMES]; + struct frame_masks *frames[MAX_CALL_FRAMES]; bool must_write_initialized; }; @@ -151,50 +180,118 @@ static int relative_idx(struct func_instance *instance, u32 insn_idx) return insn_idx - instance->subprog_start; } -static struct per_frame_masks *get_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) +static u32 frame_mask_bits(struct frame_masks *fm) { - if (!instance->frames[frame]) + return fm->words * BITS_PER_LONG; +} + +static size_t frame_mask_words(struct func_instance *instance, u32 words) +{ + return (size_t)instance->insn_cnt * FM_MASK_CNT * words; +} + +/* Mask @kind of the instruction at relative index @rel */ +static unsigned long *rel_mask(struct frame_masks *fm, u32 rel, u32 kind) +{ + return fm->bits + ((size_t)rel * FM_MASK_CNT + kind) * fm->words; +} + +/* + * Make sure @frame has a mask array at least @words wide, allocating it or + * copying the existing masks into the wider stride as needed. + * @words must be in range [1, FRAME_MAX_WORDS]. + */ +static struct frame_masks *widen_frame_masks(struct func_instance *instance, + u32 frame, u32 words) +{ + struct frame_masks *old = instance->frames[frame], *new; + u32 i, kind; + + if (old && old->words >= words) + return old; + new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), GFP_KERNEL_ACCOUNT); + if (!new) return NULL; - - return &instance->frames[frame][relative_idx(instance, insn_idx)]; -} - -static struct per_frame_masks *alloc_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) -{ - struct per_frame_masks *arr; - - if (!instance->frames[frame]) { - arr = kvzalloc_objs(*arr, instance->insn_cnt, - GFP_KERNEL_ACCOUNT); - instance->frames[frame] = arr; - if (!arr) - return ERR_PTR(-ENOMEM); + new->words = words; + if (old) { + for (i = 0; i < instance->insn_cnt; i++) + for (kind = 0; kind < FM_MASK_CNT; kind++) + memcpy(rel_mask(new, i, kind), rel_mask(old, i, kind), + old->words * sizeof(*old->bits)); + kvfree(old); } - return get_frame_masks(instance, frame, insn_idx); + instance->frames[frame] = new; + return new; } -/* Accumulate may_read masks for @frame at @insn_idx */ -static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* + * Set the inclusive half-slot range [lo, hi] in mask @kind of @frame at @insn_idx. + * An empty range, including one with a negative @hi as computed for a write + * that does not fully cover any half-slot, marks nothing. + */ +static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_idx, + u32 kind, s32 lo, s32 hi) { - struct per_frame_masks *masks; + struct frame_masks *fm; - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->may_read = spis_or(masks->may_read, mask); + /* + * An access past the frame bottom is rejected by the main verifier + * pass later, liveness only has to avoid running off the masks. + */ + hi = min_t(s32, hi, FRAME_HALF_SPIS - 1); + if (lo > hi) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1)); + if (!fm) + return -ENOMEM; + bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1); return 0; } -static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* Accumulate may_read for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) { - struct per_frame_masks *masks; + return mark_stack_range(instance, frame, insn_idx, FM_MAY_READ, lo, hi); +} - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->must_write = spis_or(masks->must_write, mask); +/* Accumulate must_write for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi); +} + +/* + * Mark every half-slot of @frame as possibly read by @insn_idx. This widens + * the masks to the maximum width: a full read recorded at a narrower width + * would leave the bits added by a later widening clear and lose part of it. + */ +static int mark_stack_read_all(struct func_instance *instance, u32 frame, u32 insn_idx) +{ + return mark_stack_read(instance, frame, insn_idx, 0, FRAME_HALF_SPIS - 1); +} + +/* Accumulate @src, a mask @src_words wide, into may_read of @frame at @insn_idx */ +static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, + const unsigned long *src, u32 src_words) +{ + u32 nbits = src_words * BITS_PER_LONG; + struct frame_masks *fm; + unsigned long *dst; + u32 last, w; + + last = find_last_bit(src, nbits); + if (last == nbits) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1)); + if (!fm) + return -ENOMEM; + dst = rel_mask(fm, relative_idx(instance, insn_idx), FM_MAY_READ); + /* @src has no bits set past @last, hence none past @fm->words either */ + src_words = min(src_words, fm->words); + for (w = 0; w < src_words; w++) + dst[w] |= src[w]; return 0; } @@ -272,33 +369,42 @@ __diag_pop(); static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { - spis_t new_before, new_after; - struct per_frame_masks *insn, *succ_insn; + unsigned long new_after[FRAME_MAX_WORDS] = {}; + unsigned long *may_read, *must_write, *live_before; + struct frame_masks *fm = instance->frames[frame]; + u32 rel = relative_idx(instance, insn_idx); struct bpf_iarray *succ; - u32 s; - bool changed; + bool changed = false; + u32 s, w; succ = bpf_insn_successors(env, insn_idx); if (succ->cnt == 0) return false; - changed = false; - insn = get_frame_masks(instance, frame, insn_idx); - new_before = SPIS_ZERO; - new_after = SPIS_ZERO; + /* All instructions of one frame array share the same mask width */ for (s = 0; s < succ->cnt; ++s) { - succ_insn = get_frame_masks(instance, frame, succ->items[s]); - new_after = spis_or(new_after, succ_insn->live_before); + unsigned long *succ_live; + + succ_live = rel_mask(fm, relative_idx(instance, succ->items[s]), FM_LIVE_BEFORE); + for (w = 0; w < fm->words; w++) + new_after[w] |= succ_live[w]; } + may_read = rel_mask(fm, rel, FM_MAY_READ); + must_write = rel_mask(fm, rel, FM_MUST_WRITE); + live_before = rel_mask(fm, rel, FM_LIVE_BEFORE); /* * New "live_before" is a union of all "live_before" of successors * minus slots written by instruction plus slots read by instruction. * new_before = (new_after & ~insn->must_write) | insn->may_read */ - new_before = spis_or(spis_and(new_after, spis_not(insn->must_write)), - insn->may_read); - changed |= !spis_equal(new_before, insn->live_before); - insn->live_before = new_before; + for (w = 0; w < fm->words; w++) { + unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w]; + + if (new_before != live_before[w]) { + live_before[w] = new_before; + changed = true; + } + } return changed; } @@ -329,10 +435,12 @@ static void update_instance(struct bpf_verifier_env *env, struct func_instance * static bool is_live_before(struct func_instance *instance, u32 insn_idx, u32 frameno, u32 half_spi) { - struct per_frame_masks *masks; + struct frame_masks *fm = instance->frames[frameno]; - masks = get_frame_masks(instance, frameno, insn_idx); - return masks && spis_test_bit(masks->live_before, half_spi); + /* No recorded access reaches past the masks, so nothing there is live */ + if (!fm || half_spi >= frame_mask_bits(fm)) + return false; + return test_bit(half_spi, rel_mask(fm, relative_idx(instance, insn_idx), FM_LIVE_BEFORE)); } int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st) @@ -447,17 +555,19 @@ static int spi_off(int spi) * When only one half is set, print as "-4h","-8h",... * Runs of 3+ consecutive fully-set SPIs are collapsed: "fp0-8..-24" */ -static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis) +static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, + const unsigned long *spis, u32 words) { int buf_sz = sizeof(env->tmp_str_buf); + int spi_cnt = words * BITS_PER_LONG / 2; char *buf = env->tmp_str_buf; int spi, n, run_start; buf[0] = '\0'; - for (spi = 0; spi < STACK_SLOTS / 2 && buf_sz > 0; spi++) { - bool lo = spis_test_bit(spis, spi * 2); - bool hi = spis_test_bit(spis, spi * 2 + 1); + for (spi = 0; spi < spi_cnt && buf_sz > 0; spi++) { + bool lo = test_bit(spi * 2, spis); + bool hi = test_bit(spi * 2 + 1, spis); const char *space = first ? "" : " "; if (!lo && !hi) @@ -467,16 +577,16 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, /* half-spi */ n = scnprintf(buf, buf_sz, "%sfp%d%d%s", space, frame, spi_off(spi) + (lo ? STACK_SLOT_SZ : 0), "h"); - } else if (spi + 2 < STACK_SLOTS / 2 && - spis_test_bit(spis, spi * 2 + 2) && - spis_test_bit(spis, spi * 2 + 3) && - spis_test_bit(spis, spi * 2 + 4) && - spis_test_bit(spis, spi * 2 + 5)) { + } else if (spi + 2 < spi_cnt && + test_bit(spi * 2 + 2, spis) && + test_bit(spi * 2 + 3, spis) && + test_bit(spi * 2 + 4, spis) && + test_bit(spi * 2 + 5, spis)) { /* 3+ consecutive full spis */ run_start = spi; - while (spi + 1 < STACK_SLOTS / 2 && - spis_test_bit(spis, (spi + 1) * 2) && - spis_test_bit(spis, (spi + 1) * 2 + 1)) + while (spi + 1 < spi_cnt && + test_bit((spi + 1) * 2, spis) && + test_bit((spi + 1) * 2 + 1, spis)) spi++; n = scnprintf(buf, buf_sz, "%sfp%d%d..%d", space, frame, spi_off(run_start), spi_off(spi)); @@ -495,7 +605,8 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i { int start = env->subprog_info[instance->subprog].start; struct bpf_insn *insns = env->prog->insnsi; - struct per_frame_masks *masks; + struct frame_masks *fm; + unsigned long *mask; int len = instance->insn_cnt; int insn_idx, frame, i; bool has_use, has_def; @@ -518,10 +629,13 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i pos = env->log.end_pos; verbose(env, " use: "); for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->may_read)) + fm = instance->frames[frame]; + if (!fm) continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read)); + mask = rel_mask(fm, i, FM_MAY_READ); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, mask, fm->words)); has_use = true; } if (!has_use) @@ -529,10 +643,13 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i pos = env->log.end_pos; verbose(env, " def: "); for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->must_write)) + fm = instance->frames[frame]; + if (!fm) continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write)); + mask = rel_mask(fm, i, FM_MUST_WRITE); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, mask, fm->words)); has_def = true; } if (!has_def) @@ -601,9 +718,9 @@ static int print_instances(struct bpf_verifier_env *env) * - same frame + different offset -> offset-imprecise * - different frames -> fully-imprecise (bitmask OR) * - * At memory access sites (LDX/STX/ST), offset-imprecise marks only - * the known frame's access mask as SPIS_ALL, while fully-imprecise - * iterates bits in the bitmask and routes each frame to its target. + * At memory access sites (LDX/STX/ST), offset-imprecise marks the known + * frame as fully read, while fully-imprecise iterates bits in the bitmask + * and routes each frame to its target. */ #define MAX_ARG_OFFSETS 4 @@ -622,8 +739,15 @@ enum arg_track_state { ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */ }; -/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ -#define MAX_ARG_SPILL_SLOTS 64 +/* + * The tracker follows as many spill slots as a subprog addresses directly + * through R10, see subprog_spill_slots(). A frame's slots at a call site are + * kept in a snapshot for the callees that receive pointers into that frame. + */ +struct spill_snapshot { + u32 slots; + struct arg_track at[]; +}; /* * Combined register + stack arg tracking: R0-R10 at indices 0-10, @@ -898,13 +1022,46 @@ static void arg_padd(struct arg_track *at, s64 delta) } /* - * Convert a byte offset from FP to a callee stack slot index. + * Number of 8-byte spill slots to track for the instructions in [@start, @end): + * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always + * followed, or the deepest 8-byte stack access made directly through R10 when + * that reaches further. Spills and fills are compiled as direct R10 accesses, + * so the count covers them whatever the frame size; a fill through a derived + * pointer that reaches past it is treated as imprecise, as one past the frame + * always was. + */ +static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) +{ + struct bpf_insn *insns = env->prog->insnsi; + int deepest = 0; + + for (int i = start; i < end; i++) { + struct bpf_insn *insn = &insns[i]; + u8 class = BPF_CLASS(insn->code); + u8 base; + + if (BPF_SIZE(insn->code) != BPF_DW) + continue; + if (class == BPF_LDX) + base = insn->src_reg; + else if (class == BPF_STX || class == BPF_ST) + base = insn->dst_reg; + else + continue; + if (base == BPF_REG_FP && insn->off < 0) + deepest = max(deepest, -insn->off); + } + return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, MAX_BPF_STACK_SLOTS); +} + +/* + * Convert a byte offset from FP to one of the @nslots tracked stack slots. * Returns -1 if out of range or not 8-byte aligned. * Slot 0 = fp-8, slot 1 = fp-16, ..., slot 7 = fp-64, .... */ -static int fp_off_to_slot(s16 off) +static int fp_off_to_slot(s16 off, u32 nslots) { - if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8)) + if (off >= 0 || off < -(int)(nslots * 8)) return -1; if (off % 8) return -1; @@ -913,7 +1070,7 @@ static int fp_off_to_slot(s16 off) static struct arg_track fill_from_stack(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, + struct arg_track *at_stack_out, u32 nslots, int depth) { struct arg_track imp = { @@ -924,7 +1081,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); return slot >= 0 ? at_stack_out[slot] : imp; } @@ -937,7 +1094,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) return imp; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) return imp; result = __arg_track_join(result, at_stack_out[slot]); @@ -952,7 +1109,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, * When exact offset is unknown conservatively add reg values to all slots in at_stack_out. */ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, - int reg, struct arg_track *at_stack_out, + int reg, struct arg_track *at_stack_out, u32 nslots, struct arg_track *val, u32 sz) { struct arg_track none = { .frame = ARG_NONE }; @@ -960,7 +1117,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); if (slot >= 0) at_stack_out[slot] = new_val; @@ -968,7 +1125,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, } cnt = at_out[reg].off_cnt; if (cnt == 0) { - for (int slot = 0; slot < MAX_ARG_SPILL_SLOTS; slot++) + for (int slot = 0; slot < nslots; slot++) at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val); return; } @@ -978,7 +1135,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) continue; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) continue; if (cnt == 1) @@ -992,16 +1149,17 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, * Clear all tracked callee stack slots overlapping the byte range * [off, off+sz-1] where off is a negative FP-relative offset. */ -static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt) +static void clear_overlapping_stack_slots(struct arg_track *at_stack, u32 nslots, s16 off, + u32 sz, int cnt) { struct arg_track none = { .frame = ARG_NONE }; if (cnt == 0) { - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) + for (int i = 0; i < nslots; i++) at_stack[i] = __arg_track_join(at_stack[i], none); return; } - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (int i = 0; i < nslots; i++) { int slot_start = -((i + 1) * 8); int slot_end = slot_start + 8; @@ -1019,33 +1177,33 @@ static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u */ static void clear_stack_for_all_offs(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, u32 sz) + struct arg_track *at_stack_out, u32 nslots, u32 sz) { int cnt, i; if (reg == BPF_REG_FP) { - clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1); + clear_overlapping_stack_slots(at_stack_out, nslots, insn->off, sz, 1); return; } cnt = at_out[reg].off_cnt; if (cnt == 0) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, cnt); return; } for (i = 0; i < cnt; i++) { s16 fp_off; if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, 0); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, 0); break; } - clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, fp_off, sz, cnt); } } static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx, struct arg_track *at_in, struct arg_track *at_stack_in, - struct arg_track *at_out, struct arg_track *at_stack_out) + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots) { bool printed = false; int i; @@ -1077,7 +1235,7 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]); } - for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (i = 0; i < nslots; i++) { if (arg_track_eq(&at_stack_out[i], &at_stack_in[i])) continue; if (!printed) { @@ -1105,7 +1263,7 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) */ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, - struct arg_track *at_out, struct arg_track *at_stack_out, + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots, const struct arg_track *at_stack_arg_entry, struct func_instance *instance, u32 *callsites) @@ -1189,14 +1347,15 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, * ARG_IMPRECISE (off_cnt == 0). */ if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, depth); + *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, nslots, + depth); } else if (src->frame >= 0 && src->frame < depth && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - struct arg_track *parent_stack = + struct spill_snapshot *parent = env->callsite_at_stack[callsites[src->frame]]; *dst = fill_from_stack(insn, at_out, insn->src_reg, - parent_stack, src->frame); + parent->at, parent->slots, src->frame); } else if (src->frame == ARG_IMPRECISE && !(src->mask & BIT(depth)) && src->mask && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { @@ -1218,12 +1377,12 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) spill_to_stack(insn, at_out, insn->dst_reg, - at_stack_out, src, sz); + at_stack_out, nslots, src, sz); if (BPF_MODE(insn->code) == BPF_ATOMIC) { if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ) clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + at_stack_out, nslots, sz); r = bpf_atomic_load_reg(insn); if (r >= 0) @@ -1236,7 +1395,7 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, /* BPF_ST to FP-derived dst: clear overlapping stack slots */ if (dst_is_local_fp) clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + at_stack_out, nslots, sz); } } @@ -1252,7 +1411,6 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, s64 access_bytes, u32 frame, u32 insn_idx) { s32 slot_hi, slot_lo; - spis_t mask; if (fp_off >= 0) /* @@ -1264,27 +1422,19 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, if (access_bytes == S64_MIN) { /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - mask = SPIS_ZERO; - spis_or_range(&mask, 0, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); + return mark_stack_read(instance, frame, insn_idx, 0, slot_hi); } if (access_bytes > 0) { /* Mark any touched slot as use */ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); + return mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); } else if (access_bytes < 0) { /* Mark only fully covered slots as def */ access_bytes = -access_bytes; slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); - if (slot_lo <= slot_hi) { - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_write(instance, frame, insn_idx, mask); - } + return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi); } return 0; } @@ -1303,7 +1453,7 @@ static int record_stack_access(struct func_instance *instance, return 0; if (arg->off_cnt == 0) { if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read(instance, frame, insn_idx, SPIS_ALL); + return mark_stack_read_all(instance, frame, insn_idx); return 0; } if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) @@ -1331,7 +1481,7 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + err = mark_stack_read_all(instance, f, insn_idx); if (err) return err; } @@ -1427,7 +1577,7 @@ static int record_arg_access(struct bpf_verifier_env *env, bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + err = mark_stack_read_all(instance, f, insn_idx); if (err) return err; } @@ -1542,7 +1692,7 @@ struct subprog_at_info { static void print_subprog_arg_access(struct bpf_verifier_env *env, int subprog, struct subprog_at_info *info, - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS]) + struct arg_track *at_stack_in, u32 nslots) { struct bpf_insn *insns = env->prog->insnsi; int start = env->subprog_info[subprog].start; @@ -1575,8 +1725,8 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, has_extra = true; } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - if (arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) + if (arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) has_extra = true; } @@ -1606,11 +1756,11 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) { - if (!arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) { + if (!arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) continue; verbose(env, " fp%+d=", -(r + 1) * 8); - verbose_arg_track(env, &at_stack_in[i][r]); + verbose_arg_track(env, &at_stack_in[(size_t)i * nslots + r]); } } @@ -1643,9 +1793,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; + u32 nslots = subprog_spill_slots(env, start, end); struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; struct arg_track at_out[MAX_AT_TRACK_REGS]; - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL; + struct arg_track *at_stack_in = NULL; struct arg_track *at_stack_out = NULL; struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS]; struct arg_track unvisited = { .frame = ARG_UNVISITED }; @@ -1657,19 +1808,19 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (!at_in) goto err_free; - at_stack_in = kvmalloc_objs(*at_stack_in, len, GFP_KERNEL_ACCOUNT); + at_stack_in = kvmalloc_objs(*at_stack_in, (size_t)len * nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_in) goto err_free; - at_stack_out = kvmalloc_objs(*at_stack_out, MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); + at_stack_out = kvmalloc_objs(*at_stack_out, nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_out) goto err_free; for (i = 0; i < len; i++) { for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[i][r] = unvisited; - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[i][r] = unvisited; + for (r = 0; r < nslots; r++) + at_stack_in[(size_t)i * nslots + r] = unvisited; } for (r = 0; r < MAX_AT_TRACK_REGS; r++) @@ -1685,8 +1836,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env, } /* Entry: all stack slots are ARG_NONE */ - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[0][r] = none; + for (r = 0; r < nslots; r++) + at_stack_in[r] = none; /* Entry: incoming stack args from caller, or ARG_NONE for main */ for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) @@ -1708,11 +1859,12 @@ redo: continue; memcpy(at_out, at_in[i], sizeof(at_out)); - memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out)); + memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out)); - arg_track_xfer(env, insn, idx, at_out, at_stack_out, + arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots, at_stack_arg_entry, instance, callsites); - arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out); + arg_track_log(env, insn, idx, at_in[i], &at_stack_in[(size_t)i * nslots], at_out, + at_stack_out, nslots); /* Propagate to successors within this subprogram */ succ = bpf_insn_successors(env, idx); @@ -1729,9 +1881,10 @@ redo: changed |= arg_track_join(env, idx, target, r, &at_in[ti][r], at_out[r]); - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) + for (r = 0; r < nslots; r++) changed |= arg_track_join(env, idx, target, -r - 1, - &at_stack_in[ti][r], at_stack_out[r]); + &at_stack_in[(size_t)ti * nslots + r], + at_stack_out[r]); } } if (changed) @@ -1754,23 +1907,24 @@ redo: } if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) { + struct spill_snapshot *snap; + kvfree(env->callsite_at_stack[idx]); - env->callsite_at_stack[idx] = - kvmalloc_objs(*env->callsite_at_stack[idx], - MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); - if (!env->callsite_at_stack[idx]) { + env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, GFP_KERNEL_ACCOUNT); + snap = env->callsite_at_stack[idx]; + if (!snap) { err = -ENOMEM; goto err_free; } - memcpy(env->callsite_at_stack[idx], - at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS); + snap->slots = nslots; + memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at)); } } info->at_in = at_in; at_in = NULL; info->len = len; - print_subprog_arg_access(env, subprog, info, at_stack_in); + print_subprog_arg_access(env, subprog, info, at_stack_in, nslots); err = 0; err_free: @@ -1797,36 +1951,52 @@ static bool has_fp_args(struct arg_track *args) * may_read: union (any pass might read the slot). * must_write: intersection (only slots written on ALL passes are guaranteed). * live_before is recomputed by a subsequent update_instance() on @dst. + * + * The two instances may have settled on different mask widths for the same + * frame, so @dst is widened to cover @src first. A word only @dst has counts + * as zero on the @src side: it unions into may_read as a no-op and intersects + * must_write to empty. */ -static void merge_instances(struct func_instance *dst, struct func_instance *src) +static int merge_instances(struct func_instance *dst, struct func_instance *src) { - int f, i; + struct frame_masks *d, *s; + u32 f, i, w; for (f = 0; f <= dst->depth; f++) { - if (!src->frames[f]) { + s = src->frames[f]; + d = dst->frames[f]; + if (!s) { /* This pass didn't touch frame f — must_write intersects with empty. */ - if (dst->frames[f]) + if (d) for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(d, i, FM_MUST_WRITE), + frame_mask_bits(d)); continue; } - if (!dst->frames[f]) { + if (!d) { /* Previous pass didn't touch frame f — take src, zero must_write. */ - dst->frames[f] = src->frames[f]; + dst->frames[f] = s; src->frames[f] = NULL; for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(s, i, FM_MUST_WRITE), frame_mask_bits(s)); continue; } + d = widen_frame_masks(dst, f, s->words); + if (!d) + return -ENOMEM; for (i = 0; i < dst->insn_cnt; i++) { - dst->frames[f][i].may_read = - spis_or(dst->frames[f][i].may_read, - src->frames[f][i].may_read); - dst->frames[f][i].must_write = - spis_and(dst->frames[f][i].must_write, - src->frames[f][i].must_write); + unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ); + unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE); + unsigned long *src_read = rel_mask(s, i, FM_MAY_READ); + unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE); + + for (w = 0; w < d->words; w++) { + dst_read[w] |= w < s->words ? src_read[w] : 0; + dst_write[w] &= w < s->words ? src_write[w] : 0; + } } } + return 0; } static struct func_instance *fresh_instance(struct func_instance *src) @@ -1941,7 +2111,7 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE) continue; for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, idx, SPIS_ALL); + err = mark_stack_read_all(instance, f, idx); if (err) goto out_free; } @@ -1980,13 +2150,18 @@ static int analyze_subprog(struct bpf_verifier_env *env, /* Pull callee's entry liveness back to caller's callsite */ { u32 callee_start = callee_instance->subprog_start; - struct per_frame_masks *entry; + struct frame_masks *callee_fm; for (int f = 0; f < callee_instance->depth; f++) { - entry = get_frame_masks(callee_instance, f, callee_start); - if (!entry) + callee_fm = callee_instance->frames[f]; + if (!callee_fm) continue; - err = mark_stack_read(instance, f, idx, entry->live_before); + err = mark_stack_read_mask(instance, f, idx, + rel_mask(callee_fm, + relative_idx(callee_instance, + callee_start), + FM_LIVE_BEFORE), + callee_fm->words); if (err) goto out_free; } @@ -1994,9 +2169,11 @@ static int analyze_subprog(struct bpf_verifier_env *env, } if (prev_instance) { - merge_instances(prev_instance, instance); + err = merge_instances(prev_instance, instance); free_instance(instance); instance = prev_instance; + if (err) + return err; } update_instance(env, instance); return 0; From 9f3d900a69be7e52d82bc06577d55a33cc41c30c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:09 +0200 Subject: [PATCH 270/660] bpf: Grow the verifier id scratch on demand The id map used to compare the ids of two states, which also serves as the id stack of release_reference() and as the id set of bpf_clear_singular_ids(), is a fixed array embedded in struct bpf_verifier_env and sized for the most registers and stack slots a state can possibly hold: 1312 entries, 10 KiB, for 512-byte frames, and four times that once frames may reach 2 KiB, which pushes the env allocation from 64 KiB to 128 KiB for every program verified. States compare a few dozen ids in practice. Turn the map and the set into arrays grown on demand, starting at 64 entries and doubling, and free them with the env. A map that cannot grow treats the states as different, an id set that cannot grow keeps every id of the state, since its counts are then incomplete, and the id stack reports -ENOMEM, which release_reference() hands to its callers. The iterator destroy path used to warn on any failure of release_reference(), which could only come from a bug while the id stack could not run out of room; it now returns -ENOMEM and keeps warning about anything else. An allocation failure thus fails the load and is never unsafe. This removes the last structure whose size scaled with the stack bound and shrinks the env by 10 KiB. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-9-memxor@gmail.com --- include/linux/bpf_verifier.h | 29 ++++++++++++--------- kernel/bpf/states.c | 44 +++++++++++++++++++++---------- kernel/bpf/verifier.c | 50 +++++++++++++++++++++++++----------- 3 files changed, 83 insertions(+), 40 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d3c31105a3ac..c5f2e18b7e54 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -409,10 +409,7 @@ struct bpf_jmp_history_entry { static_assert(MAX_CALL_FRAMES <= (1 << 4)); static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); -/* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ - MAX_CALL_FRAMES) struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -863,18 +860,27 @@ struct bpf_id_pair { u32 cur; }; +/* + * Scratch map from the ids of one verifier state to those of another, also + * used as a stack of ids. Grown on demand by bpf_id_scratch_reserve(). + */ struct bpf_idmap { u32 tmp_id_gen; u32 cnt; - struct bpf_id_pair map[BPF_ID_MAP_SIZE]; + u32 cap; + struct bpf_id_pair *map; }; +struct bpf_idset_entry { + u32 id; + u32 cnt; +}; + +/* Scratch set of ids with a use count each, grown on demand */ struct bpf_idset { u32 num_ids; - struct { - u32 id; - u32 cnt; - } entries[BPF_ID_MAP_SIZE]; + u32 cap; + struct bpf_idset_entry *entries; }; /* see verifier.c:compute_scc_callchain() */ @@ -983,10 +989,8 @@ struct bpf_verifier_env { * via callx. Allocated when the first such edge is recorded. */ unsigned long *callx_edges; - union { - struct bpf_idmap idmap_scratch; - struct bpf_idset idset_scratch; - }; + struct bpf_idmap idmap_scratch; + struct bpf_idset idset_scratch; struct { int *insn_state; int *insn_stack; @@ -1252,6 +1256,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); +bool bpf_id_scratch_reserve(void **arr, u32 *cap, u32 cnt, size_t elem_size); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, int insn_flags, int spi, int frame, const u16 *linked_regs, u8 linked_regs_cnt); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 5c7169bfb4e9..a1f7b4a87502 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -335,21 +335,18 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) return false; } - /* Reached the end of known mappings; haven't seen this id before */ - if (idmap->cnt < BPF_ID_MAP_SIZE) { - map[idmap->cnt].old = old_id; - map[idmap->cnt].cur = cur_id; - idmap->cnt++; - return true; - } - /* - * idmap slots are bounded by the number of registers and stack slots. - * Since referenced dynptrs acquire intermediate references that do - * not live in either, so the map can be exhausted. Since it is unlikely, - * fail the verification by treating the states as not equivalent. + * Reached the end of known mappings; haven't seen this id before. If + * the map cannot grow, treat the states as not equivalent, which only + * costs pruning. */ - return false; + if (!bpf_id_scratch_reserve((void **)&idmap->map, &idmap->cap, idmap->cnt, sizeof(*map))) + return false; + map = idmap->map; + map[idmap->cnt].old = old_id; + map[idmap->cnt].cur = cur_id; + idmap->cnt++; + return true; } /* @@ -967,6 +964,27 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat return true; } +/* + * Make room for one more entry in an id scratch array, doubling it as needed. + * Returns false if it could not grow; callers then treat the id as unknown + * or the states as different, which is always safe. + */ +bool bpf_id_scratch_reserve(void **arr, u32 *cap, u32 cnt, size_t elem_size) +{ + u32 new_cap; + void *p; + + if (cnt < *cap) + return true; + new_cap = *cap ? *cap * 2 : 64; + p = krealloc_array(*arr, new_cap, elem_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); + if (!p) + return false; + *arr = p; + *cap = new_cap; + return true; +} + static void reset_idmap_scratch(struct bpf_verifier_env *env) { struct bpf_idmap *idmap = &env->idmap_scratch; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a7b1142b0d51..c89f89c83dfc 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1041,7 +1041,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j; + int spi, i, j, err; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1051,8 +1051,12 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) - WARN_ON_ONCE(release_reference(env, st->id)); + if (i == 0) { + err = release_reference(env, st->id); + if (err == -ENOMEM) + return err; + WARN_ON_ONCE(err); + } bpf_mark_reg_not_init(env, st); @@ -10525,8 +10529,9 @@ static int idstack_push(struct bpf_idmap *idmap, u32 id) if (idmap->map[i].old == id) return 0; - if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE)) - return -EFAULT; + if (!bpf_id_scratch_reserve((void **)&idmap->map, &idmap->cap, idmap->cnt, + sizeof(*idmap->map))) + return -ENOMEM; idmap->map[idmap->cnt++].old = id; return 0; @@ -18951,23 +18956,27 @@ static void adjust_btf_func(struct bpf_verifier_env *env) aux->func_info[i].insn_off = env->subprog_info[i].start; } -/* Find id in idset and increment its count, or add new entry */ -static void idset_cnt_inc(struct bpf_idset *idset, u32 id) +/* + * Find id in idset and increment its count, or add new entry. Returns false + * when a new id could not be recorded, which leaves the counts incomplete. + */ +static bool idset_cnt_inc(struct bpf_idset *idset, u32 id) { u32 i; for (i = 0; i < idset->num_ids; i++) { if (idset->entries[i].id == id) { idset->entries[i].cnt++; - return; + return true; } } - /* New id */ - if (idset->num_ids < BPF_ID_MAP_SIZE) { - idset->entries[idset->num_ids].id = id; - idset->entries[idset->num_ids].cnt = 1; - idset->num_ids++; - } + if (!bpf_id_scratch_reserve((void **)&idset->entries, &idset->cap, idset->num_ids, + sizeof(*idset->entries))) + return false; + idset->entries[idset->num_ids].id = id; + idset->entries[idset->num_ids].cnt = 1; + idset->num_ids++; + return true; } /* Find id in idset and return its count, or 0 if not found */ @@ -18993,6 +19002,7 @@ void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_idset *idset = &env->idset_scratch; struct bpf_func_state *func; struct bpf_reg_state *reg; + bool complete = true; idset->num_ids = 0; @@ -19001,9 +19011,17 @@ void bpf_clear_singular_ids(struct bpf_verifier_env *env, continue; if (!reg->id) continue; - idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); + complete &= idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); })); + /* + * An id that could not be recorded may be shared, and a later + * occurrence of it may have been recorded with a count of one. Without + * complete counts keep every id; clearing is only an optimization. + */ + if (!complete) + return; + bpf_for_each_reg_in_vstate(st, func, reg, ({ if (reg->type != SCALAR_VALUE) continue; @@ -22693,6 +22711,8 @@ err_free_env: kvfree(env->gotox_tmp_buf); kvfree(env->callx_edges); kvfree(env->func_ptrs); + kfree(env->idmap_scratch.map); + kfree(env->idset_scratch.entries); bpf_diag_free(env); kvfree(env); return ret; From 7fc140ba3dd95fb0d94414ebe1b52ca990c3dc3a Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:10 +0200 Subject: [PATCH 271/660] selftests/bpf: Cover the tail call caller stack depth limit A tail call from a subprog only unwinds that subprog's frame, so the verifier refuses tail calls once the frames of the callers add up to 256 bytes or more. Nothing exercised that rule. Add a pair of tests with a caller using 240 and 256 bytes of stack respectively, the latter expecting the "tail_calls are not allowed when call stack of previous frames is 256 bytes" rejection. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-10-memxor@gmail.com --- .../selftests/bpf/progs/verifier_tailcall.c | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_tailcall.c b/tools/testing/selftests/bpf/progs/verifier_tailcall.c index b4acce60fb9b..51687da97225 100644 --- a/tools/testing/selftests/bpf/progs/verifier_tailcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_tailcall.c @@ -28,4 +28,61 @@ __naked void invalid_map_for_tail_call(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__used __naked +static int subprog_tail_call(void) +{ + asm volatile (" \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * A tail call unwinds only the frame of the subprog doing it, so the + * frames of its callers stay on the stack. With up to 33 tail calls in + * a chain the verifier caps the stack those frames may add up to at + * 256 bytes. + */ +SEC("tc") +__description("tail call from subprog with 240 bytes of caller stack") +__success +__naked void tail_call_caller_stack_ok(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__description("tail call from subprog with 256 bytes of caller stack") +__failure +__msg("tail_calls are not allowed when call stack of previous frames is 256 bytes. Too large") +__naked void tail_call_caller_stack_too_large(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 256) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 76ab55b452440e1620ad837d4d23fe73aea40f2f Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:11 +0200 Subject: [PATCH 272/660] selftests/bpf: Check that narrow stack stores define no slot The stack liveness analysis records a store as a "def" only for the 4-byte half-slots it covers completely, so a one or two byte store near the top of the frame must not define anything. Add a test that reads fp-8, stores one byte at fp-1 and two bytes at fp-4, and reads fp-8 again, expecting no def mark on either store and the second read to still use fp-8. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-11-memxor@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 7a1a0670f851..f8758eb62dac 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -2839,3 +2839,25 @@ static __used __naked void imprecise_dst_spill_join_sub(void) :: __imm(bpf_get_prandom_u32) : __clobber_all); } + +/* + * A store that does not fully cover a 4-byte half-slot defines nothing, so a + * narrow store at the top of the frame must not turn any slot into a "def", + * least of all every slot of the frame: the earlier data at fp-8 stays live. + */ +SEC("socket") +__log_level(2) +__msg("0: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__msg("1: (73) *(u8 *)(r10 -1) = r0{{$}}") +__msg("2: (6b) *(u16 *)(r10 -4) = r0{{$}}") +__msg("3: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__naked void narrow_store_defines_nothing(void) +{ + asm volatile ( + "r0 = *(u64 *)(r10 - 8);" + "*(u8 *)(r10 - 1) = r0;" + "*(u16 *)(r10 - 4) = r0;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + ::: __clobber_all); +} From 10e9fa2b0b20e96ed68a1c210461bf1b1c69244c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:12 +0200 Subject: [PATCH 273/660] selftests/bpf: Check liveness merge of masks with different widths The liveness masks of a function instance are as wide as the deepest half-slot the instance was seen to access. When the same instance is analyzed again through another call site, the new pass may have settled on a different width, and merge_instances() has to widen the original before combining the two. Add a test where the first pass of a callee reads through a pointer 248 bytes into the main frame and the second one through a pointer of unknown offset, which reads the whole frame, and check that the merged result keeps the whole-frame read. The precise read stays within the first mask word on 64-bit, so the whole-frame pass is wider under every stack budget and the merge has to widen the masks; a deeper read would need two words already, which is all a 512-byte whole-frame read needs on 64-bit, and the widening would go untested there. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-12-memxor@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index f8758eb62dac..c3b08089fef1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -2861,3 +2861,54 @@ __naked void narrow_store_defines_nothing(void) "exit;" ::: __clobber_all); } + +/* + * The same callee instance is analyzed twice: the call sites are visited in + * postorder, so the second one goes first with a precise pointer 248 bytes + * into the main frame, and the first one then passes a pointer of unknown + * offset, which reads the whole frame. The precise pass stays within the + * first word of the masks on 64-bit, so the whole-frame pass is wider under + * every stack budget and merging the second into the first has to widen the + * masks while keeping the whole-frame read. + */ +SEC("socket") +__log_level(2) +__msg("stack use/def subprog#{{[0-9]+}} merge_read_all_callee (d2,cs{{[0-9]+}}):") +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-512") +__naked void merge_keeps_whole_frame_read(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r10 - 248) = r1;" + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r1 = r10;" + "r1 += -16;" + "r1 += r0;" + "call merge_read_all_mid;" + "r1 = r10;" + "r1 += -248;" + "call merge_read_all_mid;" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static __used __naked void merge_read_all_mid(void) +{ + asm volatile ( + "call merge_read_all_callee;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_read_all_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} From ecc441fc774ec3b875f4a5e0cd75ade5e902e283 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:13 +0200 Subject: [PATCH 274/660] bpf: Size the per-frame verifier structures for a 2 KiB stack The verifier keeps a few structures whose size follows the deepest frame a program may have: the backtracking and scratched-slot bitmaps, the jump history slot index and the clamp of the liveness masks. They are all expressed through MAX_BPF_STACK_SLOTS, which derives from MAX_BPF_STACK, the frame size of the interpreter. Introduce MAX_BPF_STACK_JIT, the stack budget a program may get on a JIT that can lay out frames of any size, and derive those structures from it so that a frame may be as deep as that budget. Nothing grants the budget yet, so no program verifies differently; the only visible change is that the liveness log prints a whole-frame read up to the new depth, so the three selftests matching such reads are updated. The spill tracker of the liveness analysis keeps a table entry per instruction and tracked slot, so it follows more than the 64 slots of a MAX_BPF_STACK frame only while that table stays within what 64 slots need for the largest program; a subprog of a million instructions keeps 64, one of a quarter million may track all 256. This bounds the table at its old worst case of 640 MiB instead of letting a single deep store push it past what kvmalloc() serves. The backtracking and scratched-slot bitmaps grow from one to four words per frame, a fixed few hundred bytes per verifier environment. tmp_str_buf, which formats a frame's slot list for the log, grows from 320 to 1408 bytes so that all 256 slots still fit, and the log's line buffer from 1 to 2 KiB so that a line built from it is not cut; the environment stays within its 64 KiB allocation. The liveness masks are only as wide as the stack a frame uses, so most frames cost the same as before; a frame that is read as a whole, through a pointer of unknown offset or by bpf_loop() with two callbacks, now carries masks of eight words, 192 bytes per instruction per frame instead of 48. Measured over the 5075 selftest programs, that is 0.2% of the total peak verifier memory: strobemeta_bpf_loop and pyperf600_bpf_loop grow by 11% (1.2 MiB and 0.6 MiB), a few dozen small programs by 40 to 100 KiB each, everything else is unchanged. The next patch bounds such reads by the program's budget, so this cost is only paid once a JIT grants it. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-13-memxor@gmail.com --- include/linux/bpf_verifier.h | 26 ++++++++++++------- include/linux/filter.h | 5 ++++ kernel/bpf/liveness.c | 17 ++++++++++-- .../selftests/bpf/progs/verifier_live_stack.c | 6 ++--- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c5f2e18b7e54..019322e8196b 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -19,11 +19,12 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of tmp_str_buf in bpf_verifier. - * we need at least 306 bytes to fit full stack mask representation - * (in the "-8,-16,...,-512" form) +/* + * size of tmp_str_buf in bpf_verifier. + * we need at least 1399 bytes to fit full stack mask representation + * (in the "-8,-16,...,-2048" form) */ -#define TMP_STR_BUF_LEN 320 +#define TMP_STR_BUF_LEN 1408 /* Patch buffer size */ #define INSN_BUF_SIZE 32 @@ -244,12 +245,13 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ /* - * Largest number of BPF_REG_SIZE stack slots a single frame can have. A frame - * may use any part of the MAX_BPF_STACK budget; check_max_stack_depth() - * enforces the bound on the combined depth of frames sharing the kernel stack - * and on each frame using a private stack. + * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized + * for the largest stack budget any JIT supports. A frame may use any part of + * its program's budget; check_max_stack_depth() enforces the budget on the + * combined depth of frames sharing the kernel stack and on each frame using + * a private stack. */ -#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK / BPF_REG_SIZE) +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK_JIT / BPF_REG_SIZE) /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 @@ -719,7 +721,11 @@ struct bpf_insn_aux_data { #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ -#define BPF_VERIFIER_TMP_LOG_SIZE 1024 +/* + * Longest line the verifier log can carry: a full stack mask of + * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix. + */ +#define BPF_VERIFIER_TMP_LOG_SIZE 2048 struct bpf_verifier_log { /* Logical start and end positions of a "log window" of the verifier log. diff --git a/include/linux/filter.h b/include/linux/filter.h index 4f0662e42897..fe72e71984e5 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -98,6 +98,11 @@ struct ctl_table_header; /* BPF program can access up to 512 bytes of stack space. */ #define MAX_BPF_STACK 512 +/* + * Stack budget of a program on a JIT that lays out frames of that size. + * The interpreter and JITs without such support keep MAX_BPF_STACK. + */ +#define MAX_BPF_STACK_JIT 2048 /* Helper macros for filter block array initializers. */ diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 1d83a4cf6ec5..25e95387e0e1 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -15,7 +15,7 @@ * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on, * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have. */ -#define FRAME_HALF_SPIS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) +#define FRAME_HALF_SPIS (MAX_BPF_STACK_JIT / BPF_HALF_REG_SIZE) #define FRAME_MAX_WORDS BITS_TO_LONGS(FRAME_HALF_SPIS) /* Masks tracked for each instruction of a frame */ @@ -1021,6 +1021,19 @@ static void arg_padd(struct arg_track *at, s64 delta) } } +/* + * Slots the spill tracker may follow for a subprog of @len instructions + * without its per-instruction tables costing more than they could for the + * largest program while every frame stayed within MAX_BPF_STACK: as many + * entries as 64 slots need for BPF_COMPLEXITY_LIMIT_INSNS instructions. + */ +static u32 spill_slots_affordable(int len) +{ + u32 base = MAX_BPF_STACK / BPF_REG_SIZE; + + return max_t(u32, base, base * BPF_COMPLEXITY_LIMIT_INSNS / len); +} + /* * Number of 8-byte spill slots to track for the instructions in [@start, @end): * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always @@ -1793,7 +1806,7 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; - u32 nslots = subprog_spill_slots(env, start, end); + u32 nslots = min(subprog_spill_slots(env, start, end), spill_slots_affordable(len)); struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; struct arg_track at_out[MAX_AT_TRACK_REGS]; struct arg_track *at_stack_in = NULL; diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index c3b08089fef1..a916d4049a0b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -1953,7 +1953,7 @@ static __used __naked void fwd_parent_key_to_helper(void) SEC("socket") __log_level(2) __success -__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-512 fp0-8") +__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-2048 fp0-8") __naked void helper_arg_fallback_keeps_scanning(void) { asm volatile ( @@ -2267,7 +2267,7 @@ static __used __naked void merge_leaf_read(void) SEC("socket") __log_level(2) __success -__msg("call bpf_loop#181 ; use: fp2-8..-512 fp1-8..-512 fp0-8..-512") +__msg("call bpf_loop#181 ; use: fp2-8..-2048 fp1-8..-2048 fp0-8..-2048") __naked void bpf_loop_two_callbacks(void) { asm volatile ( @@ -2874,7 +2874,7 @@ __naked void narrow_store_defines_nothing(void) SEC("socket") __log_level(2) __msg("stack use/def subprog#{{[0-9]+}} merge_read_all_callee (d2,cs{{[0-9]+}}):") -__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-512") +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-2048") __naked void merge_keeps_whole_frame_read(void) { asm volatile ( From 9cfc28c9a8e62110e4f95592a0a8e4d86a55c6c5 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:14 +0200 Subject: [PATCH 275/660] bpf: Bound program stack use by a per-program limit The verifier checks every stack access and the combined depth of a call chain against MAX_BPF_STACK, which is also the frame size of the interpreter and the frame that JITs without subprogram tail call support set up for tail-call targets. A JIT that lays out frames of any size and lets a tail-called program set up its own frame does not need that limit; it only needs the verifier to bound how much stack a program uses in total. Add bpf_jit_supports_large_stack() for a JIT to claim that, and give each program its budget through bpf_prog_stack_limit(): MAX_BPF_STACK_JIT when the JIT is requested, the program is not offloaded and the JIT supports large stacks as well as tail calls from subprograms, MAX_BPF_STACK otherwise. The latter is what lets a tail-called program set up its own frame: without it, do_misc_fixups() gives every program with tail calls a MAX_BPF_STACK frame, which a deeper frame verified against the larger budget would overrun. The verifier keeps the budget in env->stack_limit and uses it for the bounds of fixed and variable offset stack accesses, for unprivileged stack pointer arithmetic and its speculation limit, and for the combined and private stack depth checks. A frame may use any part of its program's budget. The interpreter paths keep MAX_BPF_STACK: a program whose main frame is deeper falls back to the JIT-required path of bpf_prog_select_runtime() and one with deeper subprogram frames is rejected when patching calls for the interpreter. The extra stack that may_goto and the timed may_goto instrumentation add below a frame is, as before, not counted against the budget of a JITed program and rejected past MAX_BPF_STACK for an interpreted one. Stack liveness treats a read through a pointer of unknown offset, or a call passing a frame pointer to a subprogram, as reaching the whole frame, and widens the masks of that frame to the deepest half-slot such a read can cover. Bound that by the program's budget too: no access past it is accepted, so a program kept at MAX_BPF_STACK carries masks of two words for such frames, as before, instead of the eight that MAX_BPF_STACK_JIT needs. The three selftests matching a whole-frame read in the liveness log accept either depth. bpf_clone_redirect() transmits from inside the program, and a tc egress or lwt_xmit program that redirects to its own device runs again on top of its own frame until the datapath's recursion limit drops the packet, ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program that calls bpf_clone_redirect() keeps MAX_BPF_STACK. The redirect helpers that transmit after the program has returned leave no frame behind and do not affect the budget. Nesting through other attach points is not accounted, as before. The spill tracker of the liveness analysis follows no slot past the budget either. The capability is a boolean and the budget a single constant, in the style of the other bpf_jit_supports_*() queries, rather than a per JIT size: the budget is meant to be the same everywhere it is raised, so that programs verify identically across those architectures. No JIT declares support yet, so every program keeps its 512-byte budget. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-14-memxor@gmail.com --- include/linux/bpf_verifier.h | 20 ++++++++ include/linux/filter.h | 1 + kernel/bpf/core.c | 13 +++++ kernel/bpf/liveness.c | 51 +++++++++++-------- kernel/bpf/verifier.c | 46 +++++++++++++---- .../selftests/bpf/progs/verifier_live_stack.c | 6 +-- 6 files changed, 102 insertions(+), 35 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 019322e8196b..1c787208ff82 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1021,6 +1021,8 @@ struct bpf_verifier_env { u32 prev_jmps_processed, jmps_processed; /* maximum combined stack depth */ u32 max_stack_depth; + /* stack budget of the program, see bpf_prog_stack_limit() */ + u32 stack_limit; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -1282,6 +1284,24 @@ static inline int bpf_get_spi(s32 off) return (-off - 1) / BPF_REG_SIZE; } +/* + * Stack a program may use in total: combined over the frames of a call + * chain on the kernel stack, or per frame on a private stack. Any single + * frame may reach that deep. Only a JIT that lays out such frames may go + * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose + * tail calls let the target set up its own frame: without subprogram + * tail calls, do_misc_fixups() gives every program with tail calls a + * MAX_BPF_STACK frame, which a deeper frame would overrun. + */ +static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog) +{ + /* an offloaded program never runs on the host JIT, whatever it supports */ + if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) && + bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls()) + return MAX_BPF_STACK_JIT; + return MAX_BPF_STACK; +} + /* * Return the function state a stack pointer register refers to. frameno * shares storage with other pointer metadata, so return NULL for any diff --git a/include/linux/filter.h b/include/linux/filter.h index fe72e71984e5..e42eccb0990e 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1252,6 +1252,7 @@ bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 81b6de655b9b..aa72ad3ec689 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3496,6 +3496,19 @@ bool __weak bpf_jit_supports_private_stack(void) return false; } +/* + * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. + * Its prologue, epilogue and tail call sequences must encode such frame + * sizes and a private stack must be sized from the program's depth. The + * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(), + * whose tail calls land before the target sets up its own frame; see + * bpf_prog_stack_limit(). + */ +bool __weak bpf_jit_supports_large_stack(void) +{ + return false; +} + void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) { } diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 25e95387e0e1..16fb7dc43726 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -36,9 +36,9 @@ enum { * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A * half-slot at or past @words * BITS_PER_LONG is never read by this frame, * hence never live. An instruction that may read the whole frame, such as a - * call passing a frame pointer to another subprog, widens the array to - * FRAME_MAX_WORDS, so that the read cannot lose half-slots to a later - * widening. + * call passing a frame pointer to another subprog, widens the array to the + * program's stack budget, the deepest an accepted program can reach, so + * that the read cannot lose half-slots to a later widening. */ struct frame_masks { u32 words; @@ -264,12 +264,17 @@ static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_ /* * Mark every half-slot of @frame as possibly read by @insn_idx. This widens - * the masks to the maximum width: a full read recorded at a narrower width - * would leave the bits added by a later widening clear and lose part of it. + * the masks to the program's stack budget: a full read recorded at a narrower + * width would leave the bits added by a later widening clear and lose part of + * it. No verifier state holds a slot past the budget, so no liveness query + * reaches beyond it; an access past the budget in unreachable code may still + * widen a frame's masks further, which is harmless. */ -static int mark_stack_read_all(struct func_instance *instance, u32 frame, u32 insn_idx) +static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) { - return mark_stack_read(instance, frame, insn_idx, 0, FRAME_HALF_SPIS - 1); + return mark_stack_read(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); } /* Accumulate @src, a mask @src_words wide, into may_read of @frame at @insn_idx */ @@ -1038,10 +1043,10 @@ static u32 spill_slots_affordable(int len) * Number of 8-byte spill slots to track for the instructions in [@start, @end): * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always * followed, or the deepest 8-byte stack access made directly through R10 when - * that reaches further. Spills and fills are compiled as direct R10 accesses, - * so the count covers them whatever the frame size; a fill through a derived - * pointer that reaches past it is treated as imprecise, as one past the frame - * always was. + * that reaches further, up to the program's stack budget. Spills and fills are + * compiled as direct R10 accesses, so the count covers them whatever the frame + * size; a fill through a derived pointer that reaches past it is treated as + * imprecise, as one past the frame always was. */ static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) { @@ -1064,7 +1069,8 @@ static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) if (base == BPF_REG_FP && insn->off < 0) deepest = max(deepest, -insn->off); } - return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, MAX_BPF_STACK_SLOTS); + return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, + env->stack_limit / BPF_REG_SIZE); } /* @@ -1456,7 +1462,7 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, * 'arg' is FP-derived argument to helper/kfunc or load/store that * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. */ -static int record_stack_access(struct func_instance *instance, +static int record_stack_access(struct bpf_verifier_env *env, struct func_instance *instance, const struct arg_track *arg, s64 access_bytes, u32 frame, u32 insn_idx) { @@ -1466,7 +1472,7 @@ static int record_stack_access(struct func_instance *instance, return 0; if (arg->off_cnt == 0) { if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read_all(instance, frame, insn_idx); + return mark_stack_read_all(env, instance, frame, insn_idx); return 0; } if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) @@ -1485,7 +1491,8 @@ static int record_stack_access(struct func_instance *instance, * When a pointer is ARG_IMPRECISE, conservatively mark every frame in * the bitmask as fully used. */ -static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx) +static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance, + u32 mask, u32 insn_idx) { int depth = instance->depth; int f, err; @@ -1494,7 +1501,7 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read_all(instance, f, insn_idx); + err = mark_stack_read_all(env, instance, f, insn_idx); if (err) return err; } @@ -1563,9 +1570,9 @@ static int record_load_store_access(struct bpf_verifier_env *env, } if (ptr->frame >= 0 && ptr->frame <= depth) - return record_stack_access(instance, ptr, sz, ptr->frame, insn_idx); + return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) - return record_imprecise(instance, ptr->mask, insn_idx); + return record_imprecise(env, instance, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } @@ -1590,7 +1597,7 @@ static int record_arg_access(struct bpf_verifier_env *env, bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { - err = mark_stack_read_all(instance, f, insn_idx); + err = mark_stack_read_all(env, instance, f, insn_idx); if (err) return err; } @@ -1600,9 +1607,9 @@ static int record_arg_access(struct bpf_verifier_env *env, return 0; if (frame >= 0 && frame <= depth) - err = record_stack_access(instance, at, bytes, frame, insn_idx); + err = record_stack_access(env, instance, at, bytes, frame, insn_idx); else if (frame == ARG_IMPRECISE) - err = record_imprecise(instance, at->mask, insn_idx); + err = record_imprecise(env, instance, at->mask, insn_idx); return err; } @@ -2124,7 +2131,7 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE) continue; for (int f = 0; f <= depth; f++) { - err = mark_stack_read_all(instance, f, idx); + err = mark_stack_read_all(env, instance, f, idx); if (err) goto out_free; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c89f89c83dfc..865b6c6eb8dd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3670,7 +3670,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, int hist_spi = spi, hist_frame = state->frameno; struct bpf_stack_state *ss = bpf_stack_slot(state, spi); - /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, + /* + * caller checked that off % size == 0 and -env->stack_limit <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && @@ -5525,7 +5526,7 @@ process_func: if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { if (subprog_depth > env->max_stack_depth) env->max_stack_depth = subprog_depth; - if (subprog_depth > MAX_BPF_STACK) { + if (subprog_depth > env->stack_limit) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); return -EACCES; @@ -5534,7 +5535,7 @@ process_func: depth += subprog_depth; if (depth > env->max_stack_depth) env->max_stack_depth = depth; - if (depth > MAX_BPF_STACK) { + if (depth > env->stack_limit) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) total++; @@ -6535,10 +6536,11 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, return 0; } -/* Check that the stack access at the given offset is within bounds. The +/* + * Check that the stack access at the given offset is within bounds. The * maximum valid offset is -1. * - * The minimum valid offset is -MAX_BPF_STACK for writes, and + * The minimum valid offset is -env->stack_limit for writes, and * -state->allocated_stack for reads. */ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, @@ -6549,7 +6551,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, int min_valid_off; if (t == BPF_WRITE || env->allow_uninit_stack) - min_valid_off = -MAX_BPF_STACK; + min_valid_off = -(int)env->stack_limit; else min_valid_off = -state->allocated_stack; @@ -15232,7 +15234,8 @@ enum { REASON_STACK = -5, }; -static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, +static int retrieve_ptr_limit(const struct bpf_verifier_env *env, + const struct bpf_reg_state *ptr_reg, u32 *alu_limit, bool mask_to_left) { u32 max = 0, ptr_limit = 0; @@ -15244,7 +15247,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, * offset where we would need to deal with min/max bounds is * currently prohibited for unprivileged. */ - max = MAX_BPF_STACK + mask_to_left; + max = env->stack_limit + mask_to_left; ptr_limit = -ptr_reg->var_off.value; break; case PTR_TO_MAP_VALUE: @@ -15364,7 +15367,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, (opcode == BPF_SUB && !off_is_neg); } - err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); + err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left); if (err < 0) return err; @@ -15494,7 +15497,7 @@ static int check_stack_access_for_ptr_arithmetic( return -EACCES; } - if (off >= 0 || off < -MAX_BPF_STACK) { + if (off >= 0 || off < -(int)env->stack_limit) { verbose(env, "R%d stack pointer arithmetic goes out of range, " "prohibited for !root; off=%d\n", regno, off); return -EACCES; @@ -22354,6 +22357,26 @@ out: return err; } +/* + * bpf_clone_redirect() transmits from inside the program, so a tc egress or + * lwt_xmit program that redirects to its own device runs again on top of its + * own frame, and again from there, until the datapath's recursion limit drops + * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack + * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program + * calling it keeps the smaller budget. The redirect helpers that transmit + * after the program has returned leave no frame behind. + */ +static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + int i; + + for (i = 0; i < prog->len; i++, insn++) + if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect) + return true; + return false; +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -22378,6 +22401,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bt.env = env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; + env->stack_limit = bpf_prog_stack_limit(env->prog); + if (bpf_prog_reenters_datapath(env->prog)) + env->stack_limit = MAX_BPF_STACK; env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index a916d4049a0b..4736bcca55da 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -1953,7 +1953,7 @@ static __used __naked void fwd_parent_key_to_helper(void) SEC("socket") __log_level(2) __success -__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-2048 fp0-8") +__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-{{(512|2048)}} fp0-8") __naked void helper_arg_fallback_keeps_scanning(void) { asm volatile ( @@ -2267,7 +2267,7 @@ static __used __naked void merge_leaf_read(void) SEC("socket") __log_level(2) __success -__msg("call bpf_loop#181 ; use: fp2-8..-2048 fp1-8..-2048 fp0-8..-2048") +__msg("call bpf_loop#181 ; use: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}") __naked void bpf_loop_two_callbacks(void) { asm volatile ( @@ -2874,7 +2874,7 @@ __naked void narrow_store_defines_nothing(void) SEC("socket") __log_level(2) __msg("stack use/def subprog#{{[0-9]+}} merge_read_all_callee (d2,cs{{[0-9]+}}):") -__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-2048") +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-{{(512|2048)}}") __naked void merge_keeps_whole_frame_read(void) { asm volatile ( From 4fc52c1b549b8a425eb7f9786d1461d4a7941056 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:15 +0200 Subject: [PATCH 276/660] selftests/bpf: Add load conditions on the program stack limit The stack a program may use will depend on the JIT: 2 KiB where the JIT declares support for large stacks, 512 bytes elsewhere and for interpreted programs. Tests that probe the limit therefore need to know which one is in force. Add __load_if_large_stack() and __load_if_no_large_stack() to test_loader, analogous to the JIT load conditions, backed by a one-time probe that loads a program storing at fp-2048. The probe caches only the verifier's verdict on that store: a load that fails for another reason, such as a missing capability, is reported and probed again on the next call. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-15-memxor@gmail.com --- tools/testing/selftests/bpf/progs/bpf_misc.h | 3 ++ tools/testing/selftests/bpf/test_loader.c | 24 +++++++++++ tools/testing/selftests/bpf/testing_helpers.c | 41 +++++++++++++++++++ tools/testing/selftests/bpf/testing_helpers.h | 1 + 4 files changed, 69 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 2ced1d751ace..f3dbc3b59bff 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -175,6 +175,9 @@ #define __prepare_priv __test_tag("test_prepare_priv") #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") +/* Whether programs may use more than 512 bytes of stack on this kernel and JIT */ +#define __load_if_large_stack() __test_tag("stack_mode=large") +#define __load_if_no_large_stack() __test_tag("stack_mode=small") #define __stderr(msg) __test_tag("test_expect_stderr=" msg) #define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg) #define __stdout(msg) __test_tag("test_expect_stdout=" msg) diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index a6e3fcc1079c..25eeb1c1248b 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -45,6 +45,11 @@ enum load_mode { NO_JITED = 1 << 1, }; +enum stack_mode { + LARGE_STACK = 1 << 0, + SMALL_STACK = 1 << 1, +}; + struct test_subspec { char *name; char *description; @@ -70,6 +75,7 @@ struct test_spec { int mode_mask; int arch_mask; int load_mask; + int stack_mask; int linear_sz; const char *skip_reason; bool prepare_priv; @@ -425,6 +431,7 @@ static int parse_test_spec(struct test_loader *tester, int err = 0; u32 arch_mask = 0; u32 load_mask = 0; + u32 stack_mask = 0; struct btf *btf; enum arch arch; @@ -620,6 +627,16 @@ static int parse_test_spec(struct test_loader *tester, err = -EINVAL; goto cleanup; } + } else if ((val = str_has_pfx(s, "stack_mode="))) { + if (strcmp(val, "large") == 0) { + stack_mask = LARGE_STACK; + } else if (strcmp(val, "small") == 0) { + stack_mask = SMALL_STACK; + } else { + PRINT_FAIL("bad stack spec: '%s'", val); + err = -EINVAL; + goto cleanup; + } } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { err = push_disasm_msg(msg, &stderr_on_next_line, &spec->priv.stderr); @@ -659,6 +676,7 @@ static int parse_test_spec(struct test_loader *tester, spec->arch_mask = arch_mask ?: -1; spec->load_mask = load_mask ?: (JITED | NO_JITED); + spec->stack_mask = stack_mask ?: (LARGE_STACK | SMALL_STACK); if (spec->mode_mask == 0) spec->mode_mask = PRIV; @@ -1331,6 +1349,7 @@ void run_subtest(struct test_loader *tester, { struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; int current_runtime = is_jit_enabled() ? JITED : NO_JITED; + int current_stack = is_large_stack_supported() ? LARGE_STACK : SMALL_STACK; struct bpf_program *tprog = NULL, *tprog_iter; struct bpf_link *link, *links[32] = {}; struct test_spec *spec_iter; @@ -1360,6 +1379,11 @@ void run_subtest(struct test_loader *tester, return; } + if ((current_stack & spec->stack_mask) == 0) { + test__skip(); + return; + } + if (unpriv) { if (!can_execute_unpriv(tester, spec)) { test__skip(); diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index d1d60451c5bc..47fe61a1ebff 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -517,6 +517,47 @@ bool is_jit_enabled(void) return enabled; } +/* + * Whether the kernel accepts a program using more than 512 bytes of stack, + * which depends on the JIT in use. Probed once with a program that stores + * at the 2 KiB depth. Only the verifier's verdict on that store is cached: + * a load that fails for another reason, such as a missing capability, is + * reported and probed again on the next call. + */ +bool is_large_stack_supported(void) +{ + static int supported = -1; + struct bpf_insn insns[] = { + BPF_ST_MEM(BPF_DW, BPF_REG_10, -2048, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char log[1024] = {}; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log, + .log_size = sizeof(log), + .log_level = 1, + ); + int fd; + + if (supported >= 0) + return supported; + + fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, ARRAY_SIZE(insns), + &opts); + if (fd >= 0) { + close(fd); + supported = 1; + } else if (strstr(log, "invalid write to stack")) { + supported = 0; + } else { + fprintf(stderr, "%s: probe failed with errno %d, assuming 512 bytes:\n%s", + __func__, errno, log); + return false; + } + return supported; +} + int stack_mprotect(void) { void *buf; diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 1c58a2f08b64..f1505108e26a 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -59,6 +59,7 @@ struct bpf_insn; int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt); int testing_prog_flags(void); bool is_jit_enabled(void); +bool is_large_stack_supported(void); int stack_mprotect(void); /* Runs diff(1) on mismatch */ From 41fc2320aafe89905bd53e4ad34daade7c30445b Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:16 +0200 Subject: [PATCH 277/660] selftests/bpf: Give the 512-byte stack boundary tests a 2 KiB twin A number of tests pin the 512-byte stack limit: accesses just past it, call chains that add up to more than it, private stack frames and async callbacks that exceed it. Once a JIT raises the budget to 2 KiB those programs load, so mark them __load_if_no_large_stack() and add a counterpart at the 2 KiB boundary under __load_if_large_stack(), so that each kernel runs the pair that matches its budget. The combined-depth tests that were built from frames of a few hundred bytes now chain five 480-byte frames, which exceeds both budgets and keeps them valid on every architecture; the number of frames reported in the error then differs, so those messages match any count. The C tests are limited to 512 bytes per function by the compiler, hence the chains. The callx stack depth tests, whose callee frames add up to 608 bytes, chain four 480-byte frames behind the callx target for the same reason. No kernel grants the larger budget yet, so the 512-byte tests still run everywhere and the 2 KiB twins are skipped. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-16-memxor@gmail.com --- .../selftests/bpf/progs/async_stack_depth.c | 75 +++++++++++ .../bpf/progs/struct_ops_private_stack_fail.c | 47 ++++++- .../selftests/bpf/progs/test_global_func1.c | 65 ++++++++++ .../bpf/progs/test_global_func_deep_stack.c | 33 ++++- .../selftests/bpf/progs/verifier_callx.c | 61 +++++++-- .../bpf/progs/verifier_callx_rodata.c | 47 ++++++- .../selftests/bpf/progs/verifier_live_stack.c | 4 +- .../selftests/bpf/progs/verifier_raw_stack.c | 21 +++ .../selftests/bpf/progs/verifier_stack_ptr.c | 53 ++++++++ .../selftests/bpf/progs/verifier_var_off.c | 32 +++++ tools/testing/selftests/bpf/verifier/calls.c | 122 ++++++++++++++---- 11 files changed, 515 insertions(+), 45 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/async_stack_depth.c b/tools/testing/selftests/bpf/progs/async_stack_depth.c index 36734683acbd..9cd874a90b39 100644 --- a/tools/testing/selftests/bpf/progs/async_stack_depth.c +++ b/tools/testing/selftests/bpf/progs/async_stack_depth.c @@ -29,7 +29,49 @@ static int bad_timer_cb(void *map, int *key, struct bpf_timer *timer) return buf[255] + timer_cb(NULL, NULL, NULL); } +/* + * The same shapes scaled to the 2 KiB budget of JITs with large stacks. The + * compiler caps a single function at 512 bytes, so the depth comes from a + * chain of 480-byte frames. + */ +__attribute__((noinline)) +static int timer_cb_large_0(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69]; +} + +__attribute__((noinline)) +static int timer_cb_large_1(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_0(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_2(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_1(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_3(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_2(map, key, timer); +} + +/* 5 * 480 = 2400 bytes on its own */ +__attribute__((noinline)) +static int bad_timer_cb_large(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[255] + timer_cb_large_3(map, key, timer); +} + SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int pseudo_call_check(struct __sk_buff *ctx) { @@ -44,7 +86,25 @@ int pseudo_call_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, timer_cb) + buf[0]; } +/* main plus the four frames under timer_cb_large_3: 2400 bytes */ SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int pseudo_call_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + timer_cb_large_3(NULL, NULL, NULL); + return bpf_timer_set_callback(&elem->timer, timer_cb_large_3) + buf[0]; +} + +SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int async_call_root_check(struct __sk_buff *ctx) { @@ -58,4 +118,19 @@ int async_call_root_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, bad_timer_cb) + buf[0]; } +SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int async_call_root_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + return bpf_timer_set_callback(&elem->timer, bad_timer_cb_large) + buf[0]; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c index 1442728f5604..c8cb35b37867 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c @@ -4,6 +4,7 @@ #include #include #include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" char _license[] SEC("license") = "GPL"; @@ -25,6 +26,44 @@ __noinline static int subprog1(int *a) return subprog2(a, b); } +/* + * A chain of 480-byte frames under test_2, so that its call chain exceeds + * the 2 KiB budget of JITs with large stacks as well as the 512 bytes + * allowed elsewhere. The compiler caps a single function at 512 bytes, and + * the buffers are volatile so that it cannot shrink them. + */ +__noinline static int subprog_deep4(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return a[10] + b[20]; +} + +__noinline static int subprog_deep3(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep4(a) + b[20]; +} + +__noinline static int subprog_deep2(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep3(a) + b[20]; +} + +__noinline static int subprog_deep1(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep2(a) + b[20]; +} + SEC("struct_ops") int BPF_PROG(test_1) @@ -41,11 +80,13 @@ int BPF_PROG(test_1) SEC("struct_ops") int BPF_PROG(test_2) { - /* stack size 400 bytes */ - int a[100] = {}; + /* stack size 476 bytes, over 2 KiB with the four 480-byte deep subprogs */ + volatile char buf[376] = {}; + int a[25] = {}; + __sink(buf[375]); a[10] = 3; - val_j = subprog1(a); + val_j = subprog1(a) + subprog_deep1(a); return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_global_func1.c b/tools/testing/selftests/bpf/progs/test_global_func1.c index fc69ff18880d..f0eca282e0d4 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func1.c +++ b/tools/testing/selftests/bpf/progs/test_global_func1.c @@ -48,8 +48,73 @@ int f3(int val, struct __sk_buff *skb, int var) } SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 3 calls is") int global_func1(struct __sk_buff *skb) { return f0(1, skb) + f1(skb) + f2(2, skb) + f3(3, skb, 4); } + +/* + * A chain of five frames that stay under 512 bytes each but add up to more + * than the 2 KiB budget of JITs with large stacks; the chain also exceeds + * 512 bytes after two frames, so it is rejected everywhere. + */ +#define MAX_STACK_LARGE 480 + +__attribute__ ((noinline)) +int g0(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return skb->len; +} + +__attribute__ ((noinline)) +int g1(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g0(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g2(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g1(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g3(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g2(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g4(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g3(skb) + skb->len; +} + +SEC("tc") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +int global_func1_deep(struct __sk_buff *skb) +{ + return g4(skb); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c index 1b634b543b62..edb8a223a3cb 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c +++ b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c @@ -67,12 +67,30 @@ int XCAT(f, n)(unsigned long a) \ #define F_31 F_30 FN(31, 30) #define F_32 F_31 FN(32, 31) +/* Same, with a 480-byte frame, to exceed the 2 KiB budget of large stacks. */ +#define FNB(n, prev) \ +__attribute__((noinline)) \ +int XCAT(f, n)(unsigned long a) \ +{ \ + volatile char buf[480] = {}; \ + volatile long b = XCAT(f, prev)(a - 1); \ + if (!b) \ + return 0; \ + return b + buf[479] + 1; \ +} + +#define F_33 F_32 FNB(33, 32) +#define F_34 F_33 FNB(34, 33) +#define F_35 F_34 FNB(35, 34) +#define F_36 F_35 FNB(36, 35) +#define F_37 F_36 FNB(37, 36) + #define CAT2(a, b) a ## b #define XCAT2(a, b) CAT2(a, b) #define F(n) XCAT2(F_, n) -F(32) +F(37) /* Ensure that even 32 levels deep, the function verifies. */ SEC("syscall") @@ -88,8 +106,21 @@ int global_func_deep_stack_success(struct __sk_buff *skb) * the size. */ SEC("syscall") +__load_if_no_large_stack() __failure __msg("combined stack size of 34 calls") int global_func_deep_stack_fail(struct __sk_buff *skb) { return f32(123); } + +/* + * Five 480-byte frames on top of the chain: 5 * 480 + 33 * 16 = 2928 bytes, + * more than the 2 KiB budget of JITs with large stacks, and more than 512 + * bytes after the second frame everywhere else. + */ +SEC("syscall") +__failure __msg("combined stack size of {{[0-9]+}} calls") +int global_func_deep_stack_fail_large(struct __sk_buff *skb) +{ + return f37(123); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c index 238fc75ad154..ea3d3f97da31 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -626,19 +626,64 @@ static unsigned long use_stack_304(void) ); } + +/* Four 480-byte frames, deeper than any budget together with their caller */ +__naked __noinline __used +static unsigned long use_stack_480_0(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" + "exit;" + ); +} + /* stack of the callee of callx is accounted */ SEC("socket") -__failure __msg("combined stack size of 2 calls is") +__failure __msg("combined stack size of {{[0-9]+}} calls is") __naked void callx_stack_depth(void) { asm volatile ( "r0 = 0;" - "*(u64 *)(r10 - 304) = r0;" - "r2 = %[use_stack_304] ll;" + "*(u64 *)(r10 - 480) = r0;" + "r2 = %[use_stack_480_3] ll;" "callx r2;" "exit;" : - : __imm_addr(use_stack_304) + : __imm_addr(use_stack_480_3) : __clobber_all); } @@ -655,19 +700,19 @@ static unsigned long apply_stack_304(void) } /* - * The address of use_stack_304() is taken by the main prog that doesn't + * The address of use_stack_480_3() is taken by the main prog that doesn't * use stack, but it is called from apply_stack_304(). */ SEC("socket") -__failure __msg("combined stack size of 3 calls is") +__failure __msg("combined stack size of {{[0-9]+}} calls is") __naked void callx_stack_depth_nested(void) { asm volatile ( - "r1 = %[use_stack_304] ll;" + "r1 = %[use_stack_480_3] ll;" "call apply_stack_304;" "exit;" : - : __imm_addr(use_stack_304) + : __imm_addr(use_stack_480_3) : __clobber_all); } diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c index af1f9305da37..65d34769c8bf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -579,23 +579,58 @@ __naked void callx_rodata_recursion(void) ::: __clobber_all); } + +/* Four 480-byte frames, deeper than any budget together with their caller */ __naked __noinline __used -static unsigned long use_stack_304(void) +static unsigned long use_stack_480_0(void) { asm volatile ( "r0 = 0;" - "*(u64 *)(r10 - 304) = r0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" "exit;" ); } /* stack of all possible callees is accounted */ SEC("socket") -__failure __msg("combined stack size of 2 calls is") +__failure __msg("combined stack size of {{[0-9]+}} calls is") __naked void callx_rodata_stack_depth(void) { asm volatile ( - FUNC_TABLE2(tbl, ret0, use_stack_304) + FUNC_TABLE2(tbl, ret0, use_stack_480_3) "r0 = 0;" "*(u64 *)(r10 - 304) = r0;" "call %[bpf_get_prandom_u32];" @@ -613,11 +648,11 @@ __naked void callx_rodata_stack_depth(void) /* stack of a callback that is read from the data is accounted too */ SEC("socket") -__failure __msg("combined stack size of 2 calls is") +__failure __msg("combined stack size of {{[0-9]+}} calls is") __naked void callx_rodata_callback_stack_depth(void) { asm volatile ( - FUNC_TABLE2(tbl, use_stack_304, ret0) + FUNC_TABLE2(tbl, use_stack_480_3, ret0) "r0 = 0;" "*(u64 *)(r10 - 304) = r0;" "r6 = tbl_%= ll;" diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 4736bcca55da..a832df0b5bd2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -318,7 +318,7 @@ struct { } map_array SEC(".maps"); SEC("socket") -__failure __msg("invalid read from stack R2 off=-1024 size=8") +__failure __msg("invalid read from stack R2 off=-4096 size=8") __flag(BPF_F_TEST_STATE_FREQ) __naked unsigned long caller_stack_write_tail_call(void) { @@ -329,7 +329,7 @@ __naked unsigned long caller_stack_write_tail_call(void) "if r0 != 42 goto 1f;" "goto 2f;" "1:" - "*(u64 *)(r10 - 8) = -1024;" + "*(u64 *)(r10 - 8) = -4096;" "2:" "r1 = r6;" "r2 = r10;" diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index 9f0f48ecb421..0fe631411b9c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c @@ -240,6 +240,7 @@ __naked void load_bytes_spilled_regs_data(void) SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 1") +__load_if_no_large_stack() __failure __msg("invalid write to stack R3 off=-513 size=8") __naked void load_bytes_invalid_access_1(void) { @@ -257,6 +258,26 @@ __naked void load_bytes_invalid_access_1(void) : __clobber_all); } +SEC("tc") +__description("raw_stack: skb_load_bytes, invalid access 1, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R3 off=-2049 size=8") +__naked void load_bytes_invalid_access_1_large(void) +{ + asm volatile (" \ + r2 = 4; \ + r6 = r10; \ + r6 += -2049; \ + r3 = r6; \ + r4 = 8; \ + call %[bpf_skb_load_bytes]; \ + r0 = *(u64*)(r6 + 0); \ + exit; \ +" : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 2") __failure __msg("invalid write to stack R3 off=-1 size=8") diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 8e8cf8232255..3e0bea9819ca 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -235,6 +235,7 @@ __naked void to_stack_check_low_1(void) SEC("socket") __description("PTR_TO_STACK check low 2") +__load_if_no_large_stack() __success __failure_unpriv __msg_unpriv("R1 stack pointer arithmetic goes out of range") __retval(42) @@ -250,8 +251,27 @@ __naked void to_stack_check_low_2(void) " ::: __clobber_all); } +SEC("socket") +__description("PTR_TO_STACK check low 2, large stack") +__load_if_large_stack() +__success __failure_unpriv +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__retval(42) +__naked void to_stack_check_low_2_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 1) = r0; \ + r0 = *(u8*)(r1 + 1); \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("PTR_TO_STACK check low 3") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-513 size=1") __msg_unpriv("R1 stack pointer arithmetic goes out of range") __naked void to_stack_check_low_3(void) @@ -266,6 +286,23 @@ __naked void to_stack_check_low_3(void) " ::: __clobber_all); } +SEC("socket") +__description("PTR_TO_STACK check low 3, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2049 size=1") +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__naked void to_stack_check_low_3_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 0) = r0; \ + r0 = *(u8*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("PTR_TO_STACK check low 4") __failure __msg("math between fp pointer") @@ -483,6 +520,7 @@ l1_%=: r0 = 42; \ SEC("socket") __description("PTR_TO_STACK stack size > 512") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-520 size=8") __naked void stack_check_size_gt_512(void) { @@ -495,6 +533,21 @@ __naked void stack_check_size_gt_512(void) " ::: __clobber_all); } +SEC("socket") +__description("PTR_TO_STACK stack size > 2048") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2056 size=8") +__naked void stack_check_size_gt_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2056; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + #ifdef __BPF_FEATURE_MAY_GOTO SEC("socket") __description("PTR_TO_STACK stack size 512 with may_goto with jit") diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index a63e33675091..399884911ea5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -406,6 +406,7 @@ __naked void zero_sized_access_max_out_of_bound(void) SEC("lwt_in") __description("indirect variable-offset stack access, min out of bound") +__load_if_no_large_stack() __failure __msg("invalid variable-offset read from stack R2") __naked void access_min_out_of_bound(void) { @@ -433,6 +434,37 @@ __naked void access_min_out_of_bound(void) : __clobber_all); } +SEC("lwt_in") +__description("indirect variable-offset stack access, min out of bound, large stack") +__load_if_large_stack() +__failure __msg("invalid variable-offset read from stack R2") +__naked void access_min_out_of_bound_large(void) +{ + asm volatile (" \ + /* Fill the top 8 bytes of the stack */ \ + r2 = 0; \ + *(u64*)(r10 - 8) = r2; \ + /* Get an unknown value */ \ + r2 = *(u32*)(r1 + 0); \ + /* Make it small and 4-byte aligned */ \ + r2 &= 4; \ + r2 -= 2052; \ + /* \ + * add it to fp. We now have either fp-2052 or fp-2048, but\ + * we don't know which \ + */ \ + r2 += r10; \ + /* dereference it indirectly */ \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + SEC("cgroup/skb") __description("indirect variable-offset stack access, min_off < min_initialized") __success diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 8b94b87135bc..0af237c02ddf 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -1037,15 +1037,34 @@ .result = ACCEPT, }, { - "calls: stack overflow using two frames (pre-call access)", + /* + * Five 480-byte frames exceed the 2 KiB budget of JITs with large + * stacks, and two of them the 512 bytes allowed elsewhere. + */ + "calls: stack overflow using five frames (pre-call access)", .insns = { /* prog 1 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1054,15 +1073,30 @@ .result = REJECT, }, { - "calls: stack overflow using two frames (post-call access)", + "calls: stack overflow using five frames (post-call access)", .insns = { /* prog 1 */ BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1127,7 +1161,7 @@ .result = ACCEPT, }, { - "calls: stack depth check using three frames. test3", + "calls: stack depth check using five frames. test3", .insns = { /* main */ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), @@ -1135,66 +1169,104 @@ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 8), /* call B */ BPF_JMP_IMM(BPF_JGE, BPF_REG_6, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JLT, BPF_REG_1, 10, 1), BPF_EXIT_INSN(), - BPF_ST_MEM(BPF_B, BPF_REG_10, -224, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_JMP_IMM(BPF_JA, 0, 0, -3), /* B */ BPF_JMP_IMM(BPF_JGT, BPF_REG_1, 2, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -6), /* call A */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call C */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* C */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call D */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* D */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -12), /* call A */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP, - /* stack_main=64, stack_A=224, stack_B=256 - * and max(main+A, main+A+B) > 512 + /* + * every frame is 480 bytes, main+A = 960 > 512 and + * max(main+A, main+B+C+D+A) = 2400 > 2048 */ .errstr = "combined stack", .result = REJECT, }, { - "calls: stack depth check using three frames. test4", - /* void main(void) { + "calls: stack depth check using five frames. test4", + /* + * void main(void) { * func1(0); * func1(1); * func2(1); * } - * void func1(int alloc_or_recurse) { + * void funcN(int alloc_or_recurse) { N = 1..4 * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } else { - * func2(alloc_or_recurse); + * funcN+1(alloc_or_recurse); * } * } - * void func2(int alloc_or_recurse) { + * void func5(int alloc_or_recurse) { * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } * } + * main also calls func2 to func5 with 1 so that every function has a + * path allocating its 480 bytes, and the chain adds up to 2400 bytes, + * more than the 2 KiB budget of JITs with large stacks, and to 960 + * bytes after two frames, more than the 512 bytes allowed elsewhere. */ .insns = { /* main */ BPF_MOV64_IMM(BPF_REG_1, 0), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 6), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 12), /* call A */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 10), /* call A */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 7), /* call B */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 13), /* call B */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 16), /* call C */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 19), /* call D */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 22), /* call E */ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call B */ BPF_EXIT_INSN(), /* B */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call C */ + BPF_EXIT_INSN(), + /* C */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call D */ + BPF_EXIT_INSN(), + /* D */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call E */ + BPF_EXIT_INSN(), + /* E */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP, From 9f3bcf081ded168404d042012f651b28852850a1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:17 +0200 Subject: [PATCH 278/660] bpf, x86: Allow programs 2 KiB of stack The x86-64 JIT encodes frame sizes as 32-bit immediates in its prologue, epilogue and tail call sequences, a tail call pops the frame of the program making it and lands in the target's prologue before the target allocates its own frame, and private stacks are allocated from each program's depth, so nothing in it depends on frames staying within 512 bytes. Report bpf_jit_supports_large_stack(), which raises the budget of JITed programs to MAX_BPF_STACK_JIT: 2 KiB combined over a call chain, or per frame on a private stack, with no separate limit on a single frame. Interpreted programs keep 512 bytes. The worst case kernel stack use of a chain of tail calls grows accordingly: the callers of a tail call may still leave at most 256 bytes on the stack, so 33 programs can accumulate 8 KiB of dead frames below the last one, which may now use 2 KiB instead of 512 bytes, for a little over 10 KiB in total on a 16 KiB kernel stack. The per-cpu memory behind a private stack grows in proportion to the frames a program asks for, up to 2 KiB plus guards per frame. The budget does not depend on the privileges of the loader: an unprivileged program cannot call other BPF functions, so a tail call from one leaves no frame behind and its worst case is a single 2 KiB frame. Programs nested through helpers or attach points, such as a tracing program entered from a helper of a networking program, are not accounted against each other before or after this change; each level of nesting may now add up to 1.5 KiB more. The one nesting a program can force on itself, bpf_clone_redirect() to its own device, keeps that program at 512 bytes. The verifier state of a frame grows with the stack it uses, up to four times as many stack slots as before; the allocation is on demand, so only programs using deep frames pay for them. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-17-memxor@gmail.com --- Documentation/bpf/bpf_design_QA.rst | 10 +++++++--- arch/x86/net/bpf_jit_comp.c | 12 ++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst index eb19c945f4d5..be5fc4ac00d6 100644 --- a/Documentation/bpf/bpf_design_QA.rst +++ b/Documentation/bpf/bpf_design_QA.rst @@ -221,9 +221,13 @@ newer kernels. BPF programs need to change accordingly when this happens. Q: How much stack space a BPF program uses? ------------------------------------------- -A: Currently all program types are limited to 512 bytes of stack -space, but the verifier computes the actual amount of stack used -and both interpreter and most JITed code consume necessary amount. +A: A program may use up to 2 KiB of stack, combined over its call +chain, when the JIT of the architecture reports support for large +stacks (currently x86-64); a single function may use all of it, and +every frame of a program running on a private stack gets the whole +amount. Elsewhere, and whenever the interpreter is used, the limit is +512 bytes. The verifier computes the actual amount of stack used and +both interpreter and most JITed code consume necessary amount. Q: Can BPF be offloaded to HW? ------------------------------ diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 9fbef7504e51..e2e531dd1e0b 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -4520,6 +4520,18 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +/* + * Frame sizes are 32-bit immediates in the prologue, epilogue and tail call + * sequences, a tail call pops the caller's frame and lands in the target's + * prologue before the target allocates its own, and private stacks are + * allocated from the program's own depth, so MAX_BPF_STACK_JIT frames need + * nothing special. + */ +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + bool bpf_jit_supports_percpu_insn(void) { return true; From 1d5306c11da2de1db73e7a8494636174d1c7b402 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:18 +0200 Subject: [PATCH 279/660] bpf, arm64: Allow programs 2 KiB of stack The arm64 JIT sets up a frame with an add/sub immediate, which encodes up to 4 KiB directly, addresses the BPF stack relative to SP or the private stack pointer with offsets that fall back to a scratch register when they do not fit the load/store immediate, and a tail call pops the current frame and lands in the target's prologue before the target sets up its own. The exception callback reuses the frame record of the main program and does not depend on its size. Report bpf_jit_supports_large_stack() to give JITed programs the same 2 KiB budget as on x86-64. Tested on an emulated arm64 guest with programs loaded and run through the raw bpf() syscall: a 2 KiB frame, a 2 KiB frame with may_goto, a chain of four 512-byte frames, and a tail call from a program with a 2 KiB frame into another one; a 2056-byte frame and five 512-byte frames are rejected. The selftest suite was not run on arm64. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-18-memxor@gmail.com --- Documentation/bpf/bpf_design_QA.rst | 11 ++++++----- arch/arm64/net/bpf_jit_comp.c | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst index be5fc4ac00d6..98d70b8a2533 100644 --- a/Documentation/bpf/bpf_design_QA.rst +++ b/Documentation/bpf/bpf_design_QA.rst @@ -223,11 +223,12 @@ Q: How much stack space a BPF program uses? ------------------------------------------- A: A program may use up to 2 KiB of stack, combined over its call chain, when the JIT of the architecture reports support for large -stacks (currently x86-64); a single function may use all of it, and -every frame of a program running on a private stack gets the whole -amount. Elsewhere, and whenever the interpreter is used, the limit is -512 bytes. The verifier computes the actual amount of stack used and -both interpreter and most JITed code consume necessary amount. +stacks (currently x86-64 and arm64); a single function may use all of +it, and every frame of a program running on a private stack gets the +whole amount. Elsewhere, and for programs verified for the interpreter, +the limit is 512 bytes; the interpreter itself never runs a frame +larger than that. The verifier computes the actual amount of stack +used and both interpreter and most JITed code consume necessary amount. Q: Can BPF be offloaded to HW? ------------------------------ diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 44e3a9e683e8..475e70653454 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2503,6 +2503,11 @@ bool bpf_jit_supports_callx(void) return true; } +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, int bargs_off, int retval_off, int run_ctx_off, bool save_ret) From 17d6fa978c00f17069f5fd7bfcb6989c6d7c4977 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 18:57:19 +0200 Subject: [PATCH 280/660] selftests/bpf: Test the 2 KiB stack budget Add tests for the stack budget of JITs with large stack support: a single 2 KiB frame, with and without may_goto, four 512-byte frames that fit the budget and five that do not, a 512-byte frame calling a 1536-byte static subprog, global subprog or bpf_loop() callback and the same with eight bytes too many, variable offset writes reaching exactly the budget and past it, a tail call made from a 1 KiB frame, and two 2 KiB frames on a private stack, where each frame gets the whole budget. A program with a 2 KiB frame is checked to be rejected wherever the budget is 512 bytes, and a 1 KiB frame is rejected in a program calling bpf_clone_redirect() but accepted in one calling bpf_redirect(). Two tests run such programs: a tail call made from a subprog with a 1536-byte frame under a 240-byte caller into a program with a 2 KiB frame, and a struct_ops program on a private stack calling a subprog when both frames are 2 KiB. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924165740.2146806-19-memxor@gmail.com --- .../bpf/prog_tests/struct_ops_private_stack.c | 31 ++ .../selftests/bpf/prog_tests/tailcalls.c | 42 ++ .../selftests/bpf/prog_tests/verifier.c | 2 + .../progs/struct_ops_private_stack_large.c | 51 +++ .../bpf/progs/tailcall_large_stack.c | 62 +++ .../bpf/progs/verifier_large_stack.c | 425 ++++++++++++++++++ .../selftests/bpf/progs/verifier_live_stack.c | 23 + 7 files changed, 636 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c create mode 100644 tools/testing/selftests/bpf/progs/tailcall_large_stack.c create mode 100644 tools/testing/selftests/bpf/progs/verifier_large_stack.c diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index 98db9bafa44b..2b3ec2b79091 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c @@ -4,6 +4,7 @@ #include "struct_ops_private_stack.skel.h" #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#include "struct_ops_private_stack_large.skel.h" #if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) @@ -78,6 +79,34 @@ cleanup: struct_ops_private_stack_recur__destroy(skel); } +/* Two frames of 2 KiB each on the private stack */ +static void test_private_stack_large(void) +{ + struct struct_ops_private_stack_large *skel; + struct bpf_link *link; + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = struct_ops_private_stack_large__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_large__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_1); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->val, 100 + 30 + 12, "val"); + + bpf_link__destroy(link); +cleanup: + struct_ops_private_stack_large__destroy(skel); +} + static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) @@ -86,6 +115,8 @@ static void __test_struct_ops_private_stack(void) test_private_stack_fail(); if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); + if (test__start_subtest("private_stack_large")) + test_private_stack_large(); } #else static void __test_struct_ops_private_stack(void) diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index aefb46778307..88d223110ecd 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -10,6 +10,7 @@ #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" #include "tailcall_cgrp_storage_owner.skel.h" +#include "tailcall_large_stack.skel.h" #include "tailcall_cgrp_storage_no_storage.skel.h" #include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" @@ -2025,6 +2026,45 @@ out: tailcall_bpf2bpf2__destroy(skel_tc); } +/* + * test_tailcall_large_stack runs a tail call made from a subprog with a 1536 + * byte frame, under a 240-byte caller, into a program with a 2 KiB frame: + * + * entry (240) --call-> subprog_tail (1536) --tailcall-> classifier_0 (2048) + */ +static void test_tailcall_large_stack(void) +{ + struct tailcall_large_stack *skel; + int err, prog_fd, map_fd, key = 0; + char buff[128] = {}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = buff, + .data_size_in = sizeof(buff), + .repeat = 1, + ); + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = tailcall_large_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tailcall_large_stack__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.classifier_0); + map_fd = bpf_map__fd(skel->maps.jmp_table); + err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update jmp_table")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.entry), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 42 + 7, "retval"); +out: + tailcall_large_stack__destroy(skel); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -2096,4 +2136,6 @@ void test_tailcalls(void) test_tailcall_callback(); if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) test_tailcall_bpf2bpf_fexit_links(); + if (test__start_subtest("tailcall_large_stack")) + test_tailcall_large_stack(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index dc2c4180ee80..8a6d341b754a 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -64,6 +64,7 @@ #include "verifier_kfunc_uninit_multi.skel.h" #include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" +#include "verifier_large_stack.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" #include "verifier_linked_scalars.skel.h" @@ -248,6 +249,7 @@ void test_verifier_kfunc_uninit_multi(void) { RUN_TESTS(verifier_kfunc_uninit_ void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } +void test_verifier_large_stack(void) { RUN(verifier_large_stack); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c new file mode 100644 index 000000000000..94a25a2cff6e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +long val; + +/* On a private stack every frame gets the whole 2 KiB budget. */ +__used __naked +static long frame_2048_leaf(void) +{ + asm volatile (" \ + r1 = 30; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 12; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* test_1 is the member bpf_testmod requests a private stack for */ +SEC("struct_ops") +__naked int test_1(void) +{ + asm volatile (" \ + r1 = 100; \ + *(u64 *)(r10 - 2048) = r1; \ + call frame_2048_leaf; \ + r1 = *(u64 *)(r10 - 2048); \ + r0 += r1; \ + r1 = %[val] ll; \ + *(u64 *)(r1 + 0) = r0; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(val) + : __clobber_all); +} + +SEC(".struct_ops") +struct bpf_testmod_ops3 testmod_1 = { + .test_1 = (void *)test_1, +}; diff --git a/tools/testing/selftests/bpf/progs/tailcall_large_stack.c b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c new file mode 100644 index 000000000000..977197dac5d3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* The tail call target sets up a 2 KiB frame of its own and uses both of its ends. */ +SEC("tc") +__naked int classifier_0(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 7; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The frame of the subprog doing the tail call is unwound by it, so it may be + * large; only the frames of its callers stay behind and are limited to 256 + * bytes in total. Returns 1 when the tail call falls through. + */ +__used __naked +static int subprog_tail(void) +{ + asm volatile (" \ + r2 = 1; \ + *(u64 *)(r10 - 1536) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 1; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +SEC("tc") +__naked int entry(void) +{ + asm volatile (" \ + r2 = 2; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_large_stack.c b/tools/testing/selftests/bpf/progs/verifier_large_stack.c new file mode 100644 index 000000000000..c2a3ddf5ac91 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_large_stack.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" + +/* + * Programs may use MAX_BPF_STACK_JIT (2 KiB) of stack on JITs that support + * large stacks, combined over a call chain, with no separate limit on a + * single frame. Interpreted programs and other JITs keep 512 bytes. + */ + +SEC("socket") +__description("single frame of 2048 bytes") +__load_if_large_stack() +__success __success_unpriv __retval(42) +__naked void single_frame_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + r0 = *(u64*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("single frame of 2048 bytes without large stack support") +__load_if_no_large_stack() +__failure __msg("invalid write to stack R1 off=-2048 size=8") +__naked void single_frame_2048_no_large_stack(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 512) = r1; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_2(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_leaf; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_3(void) +{ + asm volatile (" \ + r1 = 3; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_2; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_4(void) +{ + asm volatile (" \ + r1 = 4; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_3; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("four frames of 512 bytes fit the 2 KiB budget") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void four_frames_of_512(void) +{ + asm volatile (" \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("five frames of 512 bytes exceed the 2 KiB budget") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is 2560. Too large") +__naked void five_frames_of_512(void) +{ + asm volatile (" \ + r1 = 5; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_1536_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void uneven_frames_fit(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void uneven_frames_exceed(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +#ifdef __BPF_FEATURE_MAY_GOTO +/* may_goto adds its counter below the frame; a JIT does not hold that against the budget */ +SEC("socket") +__description("frame of 2048 bytes with may_goto") +__load_if_large_stack() +__success __retval(42) +__naked void frame_2048_with_may_goto(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u32*)(r1 + 0) = r0; \ + may_goto l0_%=; \ + r2 = 100; \ + l0_%=: \ + exit; \ +" ::: __clobber_all); +} +#endif + +SEC("socket") +__description("variable offset write reaching 2048 bytes deep") +__load_if_large_stack() +__success +__naked void var_off_write_to_2048(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2048; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__description("variable offset write reaching 2056 bytes deep") +__load_if_large_stack() +__failure __msg("invalid variable-offset write to stack R2") +__naked void var_off_write_to_2056(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2056; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Each frame of a private stack gets the whole budget. */ +__used __naked +static void priv_stack_frame_2048(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 2048) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("kprobe") +__description("private stack: two frames of 2048 bytes") +__load_if_large_stack() +__arch_x86_64 +__arch_arm64 +__success __log_level(4) +__msg("stack depth max 2048") +__msg("subprog 0 (private_stack_two_frames) main {{.*}} stack 2048") +__msg("subprog 1 (priv_stack_frame_2048) static {{.*}} stack 2048") +__naked void private_stack_two_frames(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 2048) = r1; \ + call priv_stack_frame_2048; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* + * A tail call unwinds the frame of the program doing it, so a large main + * frame is fine; the 256-byte rule only concerns the frames of callers of a + * subprog that tail calls. + */ +SEC("tc") +__description("tail call from a 1 KiB frame") +__load_if_large_stack() +__success +__naked void tail_call_from_large_frame(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 1024) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * bpf_clone_redirect() can run the program again on top of its own frame, + * ten frames deep, so a program calling it keeps the 512-byte budget. The + * redirect that happens after the program returns does not. + */ +SEC("tc") +__description("1 KiB frame with bpf_clone_redirect keeps the 512-byte budget") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-1024 size=8") +__naked void clone_redirect_keeps_512(void) +{ + asm volatile (" \ + r6 = r1; \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = r6; \ + r2 = 1; \ + r3 = 0; \ + call %[bpf_clone_redirect]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_clone_redirect) + : __clobber_all); +} + +SEC("tc") +__description("1 KiB frame with bpf_redirect keeps the 2 KiB budget") +__load_if_large_stack() +__success +__naked void redirect_keeps_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = 1; \ + r2 = 0; \ + call %[bpf_redirect]; \ + exit; \ +" : + : __imm(bpf_redirect) + : __clobber_all); +} + +/* Global subprogs are verified on their own but share the call chain budget. */ +__used __naked int global_frame_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void global_subprog_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void global_subprog_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Callback frames are part of the chain of the helper that calls them. */ +static __naked int loop_cb_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void loop_callback_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +SEC("socket") +__description("520-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void loop_callback_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index a832df0b5bd2..16b2b1e57534 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -2912,3 +2912,26 @@ static __used __naked void merge_read_all_callee(void) "exit;" ::: __clobber_all); } + +/* + * A frame pointer spilled below fp-512 is a spill like any other: the fill + * restores its identity, and a load through it reads only the slot it names + * instead of the whole frame. + */ +SEC("socket") +__log_level(2) +__load_if_large_stack() +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8{{$}}") +__naked void spill_below_512_stays_precise(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 520) = r1;" + "r1 = *(u64 *)(r10 - 520);" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} From 7cd1f73b37ea0b8099ae42c49ee5318db9161913 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 18:21:04 +0000 Subject: [PATCH 281/660] selftests/bpf: Fix build of callx tests with GCC GCC doesn't support indirect calls in BPF: progs/verifier_callx.c: In function 'apply_op': progs/verifier_callx.c:977:1: error: indirect call in function, which are not supported by eBPF progs/verifier_callx_rodata.c: In function 'use_shape': progs/verifier_callx_rodata.c:746:1: error: indirect call in function, which are not supported by eBPF Build verifier_callx, verifier_callx_rodata and callx_rodata progs with clang only. Skip callx_rodata_lskel test when progs are built with GCC. Fixes: e9b22d50796d ("selftests/bpf: Add tests for callx") Fixes: b018e998e645 ("selftests/bpf: Add tests for callx through pointers in read-only data") Signed-off-by: Alexei Starovoitov Tested-by: Ihor Solodrai Link: https://patch.msgid.link/20260924182104.261823-1-alexei.starovoitov@gmail.com --- .../bpf/prog_tests/callx_rodata_lskel.c | 6 ++++ .../selftests/bpf/progs/callx_rodata.c | 28 +++++++++++++++++-- .../selftests/bpf/progs/verifier_callx.c | 3 +- .../bpf/progs/verifier_callx_rodata.c | 3 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c index 96f6179f0492..6a43b880d730 100644 --- a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -35,6 +35,12 @@ void test_callx_rodata_lskel(void) if (!ASSERT_OK_PTR(skel, "open")) return; + /* gcc doesn't support indirect calls */ + if (skel->rodata->skip) { + test__skip(); + goto out; + } + skel->rodata->bias = 7; if (!ASSERT_OK(callx_rodata_lskel__load(skel), "load")) diff --git a/tools/testing/selftests/bpf/progs/callx_rodata.c b/tools/testing/selftests/bpf/progs/callx_rodata.c index 07ae2d1b6b8e..e3f8335f80f8 100644 --- a/tools/testing/selftests/bpf/progs/callx_rodata.c +++ b/tools/testing/selftests/bpf/progs/callx_rodata.c @@ -1,16 +1,21 @@ // SPDX-License-Identifier: GPL-2.0 /* callx through pointers to functions in .rodata, loaded by light skeleton */ +#include #include #include -typedef int (*op_fn)(int); - /* set by user space before the programs are loaded, it's in .rodata too */ const volatile int bias = 1; int op_idx; +#ifdef __clang__ + +const volatile bool skip = false; + +typedef int (*op_fn)(int); + static __noinline int add_bias(int x) { return x + bias; @@ -41,6 +46,25 @@ int both_ops(void *ctx) return ops[1](ops[0](4)); } +#else + +/* gcc doesn't support indirect calls */ +const volatile bool skip = true; + +SEC("socket") +int select_op(void *ctx) +{ + return 0; +} + +SEC("socket") +int both_ops(void *ctx) +{ + return 0; +} + +#endif + /* no callx here */ SEC("socket") int read_bias(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c index ea3d3f97da31..bc31d0221f6a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -6,7 +6,8 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +/* gcc doesn't support indirect calls */ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && defined(__clang__) #define CALLX_INSN(DST, SRC, OFF, IMM) \ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c index 65d34769c8bf..db3169a1ddf6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -6,7 +6,8 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +/* gcc doesn't support indirect calls */ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && defined(__clang__) /* * Read-only data with pointers to functions, where the compiler puts tables From d5c101840af86a5b1451f8bd55cb8d76e1bfbb06 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 24 Sep 2026 11:42:54 -0700 Subject: [PATCH 282/660] bpf: crypto: Use AES-CBC and AES-ECB libraries BPF crypto was implemented using the lskcipher API, which doesn't seem to be going anywhere. lskcipher supports only ARC4 and block ciphers in CBC and ECB mode, and only with unoptimized implementations. Library APIs also have been found to be a much better approach, for a variety of reasons, including reduced overhead, greater flexibility, and having to be explicit about the crypto algorithms that are supported. We can safely ignore the theoretical ARC4 and non-AES block cipher support in BPF crypto as unused, which leaves AES-CBC and AES-ECB. AES-CBC was stated to be needed for decrypting packets using a homebrew UDP-based protocol (https://lore.kernel.org/r/d1cdfc23-b336-49a9-8833-29f05b5b9fec@linux.dev/). AES-ECB is used by the BPF self-tests, and it was stated to maybe be used in the future for QUIC-LB (https://lore.kernel.org/r/5f9c3aab-5339-463c-a86d-edac297e1e95@linux.dev/). Those reasons don't make much sense either, especially AES-ECB which isn't appropriate in new systems and should be dropped. Regardless, let's assume that both AES-CBC and AES-ECB need to be kept for now. There are library APIs for both of these now, which are much easier to use and more efficient. Reimplement BPF crypto on top of them, greatly simplifying the code. As part of this, the bpf_crypto_type abstraction layer is removed, as it's not useful. This removes the only user of the lskcipher API, so follow-up patches will be able to remove that code as well. Signed-off-by: Eric Biggers Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924184254.142720-1-ebiggers@kernel.org --- MAINTAINERS | 2 - crypto/Makefile | 3 - crypto/bpf_crypto_skcipher.c | 83 --------- include/linux/bpf_crypto.h | 24 --- kernel/bpf/Kconfig | 9 + kernel/bpf/Makefile | 4 +- kernel/bpf/crypto.c | 288 +++++++++++++---------------- kernel/bpf/verifier.c | 2 +- tools/testing/selftests/bpf/config | 2 +- 9 files changed, 136 insertions(+), 281 deletions(-) delete mode 100644 crypto/bpf_crypto_skcipher.c delete mode 100644 include/linux/bpf_crypto.h diff --git a/MAINTAINERS b/MAINTAINERS index 140eafcbbd78..cf806c2b788e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4936,8 +4936,6 @@ BPF [CRYPTO] M: Vadim Fedorenko L: bpf@vger.kernel.org S: Maintained -F: crypto/bpf_crypto_skcipher.c -F: include/linux/bpf_crypto.h F: kernel/bpf/crypto.c BPF [DOCUMENTATION] (Related to Standardization) diff --git a/crypto/Makefile b/crypto/Makefile index 8386d55a9755..33bb5ad595e8 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -22,9 +22,6 @@ crypto_skcipher-y += lskcipher.o crypto_skcipher-y += skcipher.o obj-$(CONFIG_CRYPTO_SKCIPHER2) += crypto_skcipher.o -ifeq ($(CONFIG_BPF_SYSCALL),y) -obj-$(CONFIG_CRYPTO_SKCIPHER2) += bpf_crypto_skcipher.o -endif obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o diff --git a/crypto/bpf_crypto_skcipher.c b/crypto/bpf_crypto_skcipher.c deleted file mode 100644 index a88798d3e8c8..000000000000 --- a/crypto/bpf_crypto_skcipher.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2024 Meta, Inc */ -#include -#include -#include -#include - -static void *bpf_crypto_lskcipher_alloc_tfm(const char *algo) -{ - return crypto_alloc_lskcipher(algo, 0, 0); -} - -static void bpf_crypto_lskcipher_free_tfm(void *tfm) -{ - crypto_free_lskcipher(tfm); -} - -static int bpf_crypto_lskcipher_has_algo(const char *algo) -{ - return crypto_has_skcipher(algo, CRYPTO_ALG_TYPE_LSKCIPHER, CRYPTO_ALG_TYPE_MASK); -} - -static int bpf_crypto_lskcipher_setkey(void *tfm, const u8 *key, unsigned int keylen) -{ - return crypto_lskcipher_setkey(tfm, key, keylen); -} - -static u32 bpf_crypto_lskcipher_get_flags(void *tfm) -{ - return crypto_lskcipher_get_flags(tfm); -} - -static unsigned int bpf_crypto_lskcipher_ivsize(void *tfm) -{ - return crypto_lskcipher_ivsize(tfm); -} - -static unsigned int bpf_crypto_lskcipher_statesize(void *tfm) -{ - return crypto_lskcipher_statesize(tfm); -} - -static int bpf_crypto_lskcipher_encrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_encrypt(tfm, src, dst, len, siv); -} - -static int bpf_crypto_lskcipher_decrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_decrypt(tfm, src, dst, len, siv); -} - -static const struct bpf_crypto_type bpf_crypto_lskcipher_type = { - .alloc_tfm = bpf_crypto_lskcipher_alloc_tfm, - .free_tfm = bpf_crypto_lskcipher_free_tfm, - .has_algo = bpf_crypto_lskcipher_has_algo, - .setkey = bpf_crypto_lskcipher_setkey, - .encrypt = bpf_crypto_lskcipher_encrypt, - .decrypt = bpf_crypto_lskcipher_decrypt, - .ivsize = bpf_crypto_lskcipher_ivsize, - .statesize = bpf_crypto_lskcipher_statesize, - .get_flags = bpf_crypto_lskcipher_get_flags, - .owner = THIS_MODULE, - .name = "skcipher", -}; - -static int __init bpf_crypto_skcipher_init(void) -{ - return bpf_crypto_register_type(&bpf_crypto_lskcipher_type); -} - -static void __exit bpf_crypto_skcipher_exit(void) -{ - int err = bpf_crypto_unregister_type(&bpf_crypto_lskcipher_type); - WARN_ON_ONCE(err); -} - -module_init(bpf_crypto_skcipher_init); -module_exit(bpf_crypto_skcipher_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Symmetric key cipher support for BPF"); diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h deleted file mode 100644 index a41e71d4e2d9..000000000000 --- a/include/linux/bpf_crypto.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ -#ifndef _BPF_CRYPTO_H -#define _BPF_CRYPTO_H - -struct bpf_crypto_type { - void *(*alloc_tfm)(const char *algo); - void (*free_tfm)(void *tfm); - int (*has_algo)(const char *algo); - int (*setkey)(void *tfm, const u8 *key, unsigned int keylen); - int (*setauthsize)(void *tfm, unsigned int authsize); - int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - unsigned int (*ivsize)(void *tfm); - unsigned int (*statesize)(void *tfm); - u32 (*get_flags)(void *tfm); - struct module *owner; - char name[14]; -}; - -int bpf_crypto_register_type(const struct bpf_crypto_type *type); -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type); - -#endif /* _BPF_CRYPTO_H */ diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index d7d25477ef48..98493e32db2a 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -91,6 +91,15 @@ config BPF_UNPRIV_DEFAULT_OFF If you are unsure how to answer this question, answer Y. +config BPF_CRYPTO + def_bool y + depends on BPF_SYSCALL + select CRYPTO_LIB_AES_CBC + select CRYPTO_LIB_AES_ECB + help + Provide the kfuncs needed for BPF programs to encrypt and decrypt + data. The supported algorithms are AES-CBC and AES-ECB. + source "kernel/bpf/preload/Kconfig" config BPF_LSM diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 9a92c348bbda..c1f9b0d3468d 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -58,9 +58,7 @@ obj-$(CONFIG_BPF_SYSCALL) += cpumask.o # semantics within pahole are revisited accordingly. obj-${CONFIG_BPF_LSM} += bpf_lsm_proto.o bpf_lsm.o endif -ifneq ($(CONFIG_CRYPTO),) -obj-$(CONFIG_BPF_SYSCALL) += crypto.o -endif +obj-$(CONFIG_BPF_CRYPTO) += crypto.o obj-$(CONFIG_BPF_PRELOAD) += preload/ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 3f3fe2450fc6..69bc29d0a4d4 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2024 Meta, Inc */ #include -#include #include #include #include #include -#include #include -#include - -struct bpf_crypto_type_list { - const struct bpf_crypto_type *type; - struct list_head list; -}; +#include +#include /* BPF crypto initialization parameters struct */ /** @@ -36,94 +30,53 @@ struct bpf_crypto_params { u32 authsize; }; -static LIST_HEAD(bpf_crypto_types); -static DECLARE_RWSEM(bpf_crypto_types_sem); +enum bpf_crypto_algo_id { + BPF_ALGO_AES_CBC, + BPF_ALGO_AES_ECB, +}; + +static const struct { + const char *type_name; + const char *algo_name; + enum bpf_crypto_algo_id algo; +} bpf_crypto_algos[] = { + { "skcipher", "cbc(aes)", BPF_ALGO_AES_CBC }, + { "skcipher", "ecb(aes)", BPF_ALGO_AES_ECB }, +}; + +static bool bpf_crypto_find_algo(const struct bpf_crypto_params *params, + enum bpf_crypto_algo_id *id_ret) +{ + for (size_t i = 0; i < ARRAY_SIZE(bpf_crypto_algos); i++) { + if (strncmp(bpf_crypto_algos[i].type_name, params->type, + sizeof(params->type)) == 0 && + strncmp(bpf_crypto_algos[i].algo_name, params->algo, + sizeof(params->algo)) == 0) { + *id_ret = bpf_crypto_algos[i].algo; + return true; + } + } + return false; +} /** * struct bpf_crypto_ctx - refcounted BPF crypto context structure - * @type: The pointer to bpf crypto type - * @tfm: The pointer to instance of crypto API struct. - * @siv_len: Size of IV and state storage for cipher + * @key: The crypto key + * @algo: The crypto algorithm ID * @rcu: The RCU head used to free the crypto context with RCU safety. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. */ struct bpf_crypto_ctx { - const struct bpf_crypto_type *type; - void *tfm; - u32 siv_len; + union { + struct aes_key aes; + } key; + enum bpf_crypto_algo_id algo; struct rcu_head rcu; refcount_t usage; }; -int bpf_crypto_register_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -EBUSY; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (!strcmp(node->type->name, type->name)) - goto unlock; - } - - node = kmalloc_obj(*node); - err = -ENOMEM; - if (!node) - goto unlock; - - node->type = type; - list_add(&node->list, &bpf_crypto_types); - err = 0; - -unlock: - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_register_type); - -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -ENOENT; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, type->name)) - continue; - - list_del(&node->list); - kfree(node); - err = 0; - break; - } - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_unregister_type); - -static const struct bpf_crypto_type *bpf_crypto_get_type(const char *name) -{ - const struct bpf_crypto_type *type = ERR_PTR(-ENOENT); - struct bpf_crypto_type_list *node; - - down_read(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, name)) - continue; - - if (try_module_get(node->type->owner)) - type = node->type; - break; - } - up_read(&bpf_crypto_types_sem); - - return type; -} - __bpf_kfunc_start_defs(); /** @@ -132,93 +85,70 @@ __bpf_kfunc_start_defs(); * Allocates a crypto context that can be used, acquired, and released by * a BPF program. The crypto context returned by this function must either * be embedded in a map as a kptr, or freed with bpf_crypto_ctx_release(). - * As crypto API functions use GFP_KERNEL allocations, this function can - * only be used in sleepable BPF programs. + * As this uses a GFP_KERNEL allocation, this function can only be used in + * sleepable BPF programs. * * bpf_crypto_ctx_create() allocates memory for crypto context. * It may return NULL if no memory is available. * @params: pointer to struct bpf_crypto_params which contains all the * details needed to initialise crypto context. - * @params__sz: size of steuct bpf_crypto_params usef by bpf program - * @err: integer to store error code when NULL is returned. + * @params__sz: size of struct bpf_crypto_params used by bpf program + * @err_ret: integer to store error code when NULL is returned. */ __bpf_kfunc struct bpf_crypto_ctx * bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, - int *err) + int *err_ret) { - const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; + int err; if (!params || params__sz != sizeof(struct bpf_crypto_params) || params->reserved[0] || params->reserved[1]) { - *err = -EINVAL; + *err_ret = -EINVAL; return NULL; } - type = bpf_crypto_get_type(params->type); - if (IS_ERR(type)) { - *err = PTR_ERR(type); - return NULL; - } - - if (!type->has_algo(params->algo)) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - - if (!!params->authsize ^ !!type->setauthsize) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - if (!params->key_len || params->key_len > sizeof(params->key)) { - *err = -EINVAL; - goto err_module_put; + *err_ret = -EINVAL; + return NULL; } ctx = kzalloc_obj(*ctx); if (!ctx) { - *err = -ENOMEM; - goto err_module_put; + *err_ret = -ENOMEM; + return NULL; } - ctx->type = type; - ctx->tfm = type->alloc_tfm(params->algo); - if (IS_ERR(ctx->tfm)) { - *err = PTR_ERR(ctx->tfm); - goto err_free_ctx; + if (!bpf_crypto_find_algo(params, &ctx->algo)) { + err = -EOPNOTSUPP; + goto out; } - if (params->authsize) { - *err = type->setauthsize(ctx->tfm, params->authsize); - if (*err) - goto err_free_tfm; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + case BPF_ALGO_AES_ECB: + if (params->authsize) + err = -EOPNOTSUPP; + else + err = aes_preparekey(&ctx->key.aes, params->key, + params->key_len); + break; + default: + WARN_ON_ONCE(1); + err = -EOPNOTSUPP; + break; } - *err = type->setkey(ctx->tfm, params->key, params->key_len); - if (*err) - goto err_free_tfm; - - if (type->get_flags(ctx->tfm) & CRYPTO_TFM_NEED_KEY) { - *err = -EINVAL; - goto err_free_tfm; +out: + if (err) { + kfree_sensitive(ctx); + *err_ret = err; + return NULL; } - - ctx->siv_len = type->ivsize(ctx->tfm) + type->statesize(ctx->tfm); - refcount_set(&ctx->usage, 1); - + *err_ret = 0; return ctx; - -err_free_tfm: - type->free_tfm(ctx->tfm); -err_free_ctx: - kfree(ctx); -err_module_put: - module_put(type->owner); - - return NULL; } static void crypto_free_cb(struct rcu_head *head) @@ -226,9 +156,7 @@ static void crypto_free_cb(struct rcu_head *head) struct bpf_crypto_ctx *ctx; ctx = container_of(head, struct bpf_crypto_ctx, rcu); - ctx->type->free_tfm(ctx->tfm); - module_put(ctx->type->owner); - kfree(ctx); + kfree_sensitive(ctx); } /** @@ -268,27 +196,53 @@ __bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) } CFI_NOSEAL(bpf_crypto_ctx_release_dtor); +static int bpf_aes_cbc_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != AES_BLOCK_SIZE) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_cbc_decrypt(dst, src, src_len, iv, &ctx->key.aes); + else + aes_cbc_encrypt(dst, src, src_len, iv, &ctx->key.aes); + return 0; +} + +static int bpf_aes_ecb_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != 0) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_ecb_decrypt(dst, src, src_len, &ctx->key.aes); + else + aes_ecb_encrypt(dst, src, src_len, &ctx->key.aes); + return 0; +} + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, - const struct bpf_dynptr_kern *siv, + const struct bpf_dynptr_kern *iv, bool decrypt) { - u32 src_len, dst_len, siv_len; + u32 src_len, dst_len, iv_len; const u8 *psrc; u8 *pdst, *piv; - int err; if (__bpf_dynptr_is_rdonly(dst)) return -EINVAL; - siv_len = siv ? __bpf_dynptr_size(siv) : 0; + iv_len = iv ? __bpf_dynptr_size(iv) : 0; src_len = __bpf_dynptr_size(src); dst_len = __bpf_dynptr_size(dst); - if (!src_len || !dst_len || src_len > dst_len) - return -EINVAL; - - if (siv_len != ctx->siv_len) + if (!src_len || !dst_len) return -EINVAL; psrc = __bpf_dynptr_data(src, src_len); @@ -298,14 +252,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, if (!pdst) return -EINVAL; - piv = siv_len ? __bpf_dynptr_data_rw(siv, siv_len) : NULL; - if (siv_len && !piv) + piv = iv_len ? __bpf_dynptr_data_rw(iv, iv_len) : NULL; + if (iv_len && !piv) return -EINVAL; - err = decrypt ? ctx->type->decrypt(ctx->tfm, psrc, pdst, src_len, piv) - : ctx->type->encrypt(ctx->tfm, psrc, pdst, src_len, piv); - - return err; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + return bpf_aes_cbc_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + case BPF_ALGO_AES_ECB: + return bpf_aes_ecb_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + default: + return -EINVAL; + } } /** @@ -313,20 +273,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the encrypted data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Decrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, true); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, true); } /** @@ -334,20 +294,20 @@ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the plain data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Encrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_encrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, false); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, false); } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 865b6c6eb8dd..4077c88d9525 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4742,7 +4742,7 @@ BTF_ID(struct, cgroup) BTF_ID(struct, bpf_cpumask) #endif BTF_ID(struct, task_struct) -#ifdef CONFIG_CRYPTO +#ifdef CONFIG_BPF_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif #ifdef CONFIG_INET diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index d292cb60a5a4..2b883b388f90 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -18,7 +18,7 @@ CONFIG_CRYPTO_SHA512=y CONFIG_CRYPTO_USER_API=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y -CONFIG_CRYPTO_SKCIPHER=y +CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_ECB=y CONFIG_CRYPTO_AES=y CONFIG_DEBUG_INFO=y From 9bf1f086e7dd74ab6da594cdb88a55dd39eb0af9 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 21:34:01 +0200 Subject: [PATCH 283/660] bpf: Return the id scratch to a fixed array sized for the stack budget Commit 9f3d900a69be ("bpf: Grow the verifier id scratch on demand") turned the id map comparing the ids of two states, which also serves as the id stack of release_reference() and as the id set of bpf_clear_singular_ids(), into arrays grown on demand, so that the env would not embed a map sized for 2 KiB frames. That gave the map a way to fail, and with it release_reference() and the iterator destroy path an -ENOMEM to hand up, states_equal() a case where states compare unequal for lack of memory, and bpf_clear_singular_ids() a flag to keep every id when the set could not record them all. The env is allocated once per program load, so the map can afford to be sized for the largest state again: MAX_BPF_STACK_SLOTS in each of MAX_CALL_FRAMES frames, which a state on a private stack can reach since every frame there may use the whole budget. That makes the map 34 KiB instead of 10 KiB, a small price per load for having no growth that can fail. Make the map and the set a fixed union in the env again and drop the growth and its failure handling. Suggested-by: Alexei Starovoitov Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/DLNQW1IRYK94.1DP0M5VKZSY32@gmail.com Link: https://patch.msgid.link/20260924193403.3032037-1-memxor@gmail.com --- include/linux/bpf_verifier.h | 29 +++++++++------------ kernel/bpf/states.c | 44 ++++++++++--------------------- kernel/bpf/verifier.c | 50 +++++++++++------------------------- 3 files changed, 40 insertions(+), 83 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1c787208ff82..646e447cebd8 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -411,7 +411,10 @@ struct bpf_jmp_history_entry { static_assert(MAX_CALL_FRAMES <= (1 << 4)); static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); +/* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -866,27 +869,18 @@ struct bpf_id_pair { u32 cur; }; -/* - * Scratch map from the ids of one verifier state to those of another, also - * used as a stack of ids. Grown on demand by bpf_id_scratch_reserve(). - */ struct bpf_idmap { u32 tmp_id_gen; u32 cnt; - u32 cap; - struct bpf_id_pair *map; + struct bpf_id_pair map[BPF_ID_MAP_SIZE]; }; -struct bpf_idset_entry { - u32 id; - u32 cnt; -}; - -/* Scratch set of ids with a use count each, grown on demand */ struct bpf_idset { u32 num_ids; - u32 cap; - struct bpf_idset_entry *entries; + struct { + u32 id; + u32 cnt; + } entries[BPF_ID_MAP_SIZE]; }; /* see verifier.c:compute_scc_callchain() */ @@ -995,8 +989,10 @@ struct bpf_verifier_env { * via callx. Allocated when the first such edge is recorded. */ unsigned long *callx_edges; - struct bpf_idmap idmap_scratch; - struct bpf_idset idset_scratch; + union { + struct bpf_idmap idmap_scratch; + struct bpf_idset idset_scratch; + }; struct { int *insn_state; int *insn_stack; @@ -1264,7 +1260,6 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); -bool bpf_id_scratch_reserve(void **arr, u32 *cap, u32 cnt, size_t elem_size); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, int insn_flags, int spi, int frame, const u16 *linked_regs, u8 linked_regs_cnt); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index a1f7b4a87502..5c7169bfb4e9 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -335,18 +335,21 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) return false; } + /* Reached the end of known mappings; haven't seen this id before */ + if (idmap->cnt < BPF_ID_MAP_SIZE) { + map[idmap->cnt].old = old_id; + map[idmap->cnt].cur = cur_id; + idmap->cnt++; + return true; + } + /* - * Reached the end of known mappings; haven't seen this id before. If - * the map cannot grow, treat the states as not equivalent, which only - * costs pruning. + * idmap slots are bounded by the number of registers and stack slots. + * Since referenced dynptrs acquire intermediate references that do + * not live in either, so the map can be exhausted. Since it is unlikely, + * fail the verification by treating the states as not equivalent. */ - if (!bpf_id_scratch_reserve((void **)&idmap->map, &idmap->cap, idmap->cnt, sizeof(*map))) - return false; - map = idmap->map; - map[idmap->cnt].old = old_id; - map[idmap->cnt].cur = cur_id; - idmap->cnt++; - return true; + return false; } /* @@ -964,27 +967,6 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat return true; } -/* - * Make room for one more entry in an id scratch array, doubling it as needed. - * Returns false if it could not grow; callers then treat the id as unknown - * or the states as different, which is always safe. - */ -bool bpf_id_scratch_reserve(void **arr, u32 *cap, u32 cnt, size_t elem_size) -{ - u32 new_cap; - void *p; - - if (cnt < *cap) - return true; - new_cap = *cap ? *cap * 2 : 64; - p = krealloc_array(*arr, new_cap, elem_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); - if (!p) - return false; - *arr = p; - *cap = new_cap; - return true; -} - static void reset_idmap_scratch(struct bpf_verifier_env *env) { struct bpf_idmap *idmap = &env->idmap_scratch; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4077c88d9525..7fd00c540cff 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1041,7 +1041,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j, err; + int spi, i, j; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1051,12 +1051,8 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) { - err = release_reference(env, st->id); - if (err == -ENOMEM) - return err; - WARN_ON_ONCE(err); - } + if (i == 0) + WARN_ON_ONCE(release_reference(env, st->id)); bpf_mark_reg_not_init(env, st); @@ -10531,9 +10527,8 @@ static int idstack_push(struct bpf_idmap *idmap, u32 id) if (idmap->map[i].old == id) return 0; - if (!bpf_id_scratch_reserve((void **)&idmap->map, &idmap->cap, idmap->cnt, - sizeof(*idmap->map))) - return -ENOMEM; + if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE)) + return -EFAULT; idmap->map[idmap->cnt++].old = id; return 0; @@ -18959,27 +18954,23 @@ static void adjust_btf_func(struct bpf_verifier_env *env) aux->func_info[i].insn_off = env->subprog_info[i].start; } -/* - * Find id in idset and increment its count, or add new entry. Returns false - * when a new id could not be recorded, which leaves the counts incomplete. - */ -static bool idset_cnt_inc(struct bpf_idset *idset, u32 id) +/* Find id in idset and increment its count, or add new entry */ +static void idset_cnt_inc(struct bpf_idset *idset, u32 id) { u32 i; for (i = 0; i < idset->num_ids; i++) { if (idset->entries[i].id == id) { idset->entries[i].cnt++; - return true; + return; } } - if (!bpf_id_scratch_reserve((void **)&idset->entries, &idset->cap, idset->num_ids, - sizeof(*idset->entries))) - return false; - idset->entries[idset->num_ids].id = id; - idset->entries[idset->num_ids].cnt = 1; - idset->num_ids++; - return true; + /* New id */ + if (idset->num_ids < BPF_ID_MAP_SIZE) { + idset->entries[idset->num_ids].id = id; + idset->entries[idset->num_ids].cnt = 1; + idset->num_ids++; + } } /* Find id in idset and return its count, or 0 if not found */ @@ -19005,7 +18996,6 @@ void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_idset *idset = &env->idset_scratch; struct bpf_func_state *func; struct bpf_reg_state *reg; - bool complete = true; idset->num_ids = 0; @@ -19014,17 +19004,9 @@ void bpf_clear_singular_ids(struct bpf_verifier_env *env, continue; if (!reg->id) continue; - complete &= idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); + idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); })); - /* - * An id that could not be recorded may be shared, and a later - * occurrence of it may have been recorded with a count of one. Without - * complete counts keep every id; clearing is only an optimization. - */ - if (!complete) - return; - bpf_for_each_reg_in_vstate(st, func, reg, ({ if (reg->type != SCALAR_VALUE) continue; @@ -22737,8 +22719,6 @@ err_free_env: kvfree(env->gotox_tmp_buf); kvfree(env->callx_edges); kvfree(env->func_ptrs); - kfree(env->idmap_scratch.map); - kfree(env->idset_scratch.entries); bpf_diag_free(env); kvfree(env); return ret; From 9774ae5951c21e2c145a88ccbf9899f727dc2e2a Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 19:06:35 +0200 Subject: [PATCH 284/660] bpf: Correct verifier diagnostic attribution for stack reads Verifier memory diagnostics currently label every fixed-offset stack read rejected by check_stack_read_fixed_off() as uninitialized. Dynptr, iterator, and IRQ-flag slots instead contain initialized verifier-managed state, so the report incorrectly suggests initialization or CAP_PERFMON. The variable-offset read without a destination register is currently reached by atomic read-modify-write instructions, but that follows from the call sites rather than the check_stack_read() interface. Its diagnostic also attributes the access to a helper. Classify rejected stack reads by slot type. Retain the existing uninitialized report for STACK_INVALID, and describe verifier-managed slots as opaque state. Use instruction-neutral wording for the variable-offset read while leaving the existing verifier messages unchanged. Reported-by: Sashiko Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260815065956.49D2B1F000E9@smtp.kernel.org/ Link: https://patch.msgid.link/20260924170646.2366016-2-memxor@gmail.com --- kernel/bpf/verifier.c | 63 ++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7fd00c540cff..7f220fbcc7e9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3979,19 +3979,46 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } -static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, - int size) +static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size, + enum bpf_stack_slot_type type) { - const char *reason; + const char *problem, *reason, *suggestion, *kind; - reason = bpf_diag_fmt(env, - "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " - "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", - size, off, i); - bpf_diag_memory( - env, env->insn_idx, "uninitialized stack read", reason, - "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " - "or load with CAP_PERFMON if uninitialized stack reads are intended."); + if (type == STACK_INVALID) { + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); + return; + } + + switch (type) { + case STACK_DYNPTR: + kind = "dynptr"; + suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly."; + break; + case STACK_ITER: + kind = "iterator"; + suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly."; + break; + case STACK_IRQ_FLAG: + kind = "IRQ flag"; + suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly."; + break; + default: + return; + } + + problem = bpf_diag_fmt(env, "direct read of %s stack state", kind); + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. " + "This state has an opaque representation that BPF programs cannot read directly.", + size, off, i, kind); + bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion); } /* Read the stack at 'off' and put the results into the register indicated by @@ -4083,7 +4110,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } @@ -4142,7 +4169,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } @@ -4240,13 +4267,13 @@ static int check_stack_read(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); - reason = bpf_diag_fmt(env, - "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " - "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + reason = bpf_diag_fmt( + env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. " + "This stack access requires a constant stack offset and a precise initialized range.", tn_buf, off, size); bpf_diag_memory( - env, env->insn_idx, "variable stack access", reason, - "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); + env, env->insn_idx, "variable-offset stack access", reason, + "Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity From e4b20ef33f93ed0dad5b8ff2d0690d83f1215a9b Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 19:06:36 +0200 Subject: [PATCH 285/660] selftests/bpf: Test verifier stack-read diagnostic attribution Verifier diagnostics distinguish uninitialized stack bytes from opaque dynptr, iterator, and IRQ-flag state, and identify variable-offset stack accesses without changing the existing verbose messages. Add output assertions to the existing uninitialized-stack, dynptr, and iterator rejection cases. The uninitialized-stack assertion uses the CAP_PERFMON-less read inside the allocated stack, because a read below the allocated stack is rejected by the bounds check before the slot classification runs. Add a direct IRQ-flag read and a variable-offset atomic stack access to cover the other classifications. Retain an assertion for the legacy helper-worded verbose message in the atomic test. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260924170646.2366016-3-memxor@gmail.com --- .../testing/selftests/bpf/progs/dynptr_fail.c | 3 +++ tools/testing/selftests/bpf/progs/irq.c | 13 +++++++++ .../selftests/bpf/progs/iters_state_safety.c | 3 +++ .../selftests/bpf/progs/verifier_spill_fill.c | 2 ++ .../selftests/bpf/progs/verifier_xadd.c | 27 +++++++++++++++++++ 5 files changed, 48 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 1cd61d72c166..9418dfe4d7b7 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -560,6 +560,9 @@ int global(void *ctx) /* A direct read should fail */ SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of dynptr stack state") +__msg("verifier-managed dynptr state") +__msg("Use dynptr helpers or kfuncs") int invalid_read1(void *ctx) { struct bpf_dynptr ptr; diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c index 53df6d248e26..50727fa9b11d 100644 --- a/tools/testing/selftests/bpf/progs/irq.c +++ b/tools/testing/selftests/bpf/progs/irq.c @@ -14,6 +14,19 @@ extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr struct bpf_res_spin_lock lockA __hidden SEC(".data.A"); struct bpf_res_spin_lock lockB __hidden SEC(".data.B"); +SEC("?tc") +__failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of IRQ flag stack state") +__msg("verifier-managed IRQ flag state") +__msg("Pass the saved IRQ flag to the matching restore kfunc") +int irq_flag_direct_read(struct __sk_buff *ctx) +{ + unsigned long flags; + + bpf_local_irq_save(&flags); + return flags; +} + SEC("?tc") __failure __msg("R1 type=map_value expected=fp") int irq_save_bad_arg(struct __sk_buff *ctx) diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c index e5bb9fe6d5e5..b2ba4609a31e 100644 --- a/tools/testing/selftests/bpf/progs/iters_state_safety.c +++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c @@ -354,6 +354,9 @@ int next_after_destroy_fail(void *ctx) SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of iterator stack state") +__msg("verifier-managed iterator state") +__msg("Use iterator kfuncs") int __naked read_from_iter_slot_fail(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 39a1766dae3f..e1a698db0364 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1277,6 +1277,8 @@ __description("stack_noperfmon: reject read of invalid slots") __success __caps_unpriv(CAP_BPF) __failure_unpriv __msg_unpriv("invalid read from stack off -8+1 size 8") +__msg_unpriv("Verification failed: Memory Safety: Uninitialized stack read") +__msg_unpriv("Initialize every byte in the stack range before reading it") __naked void stack_noperfmon_reject_invalid_read(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_xadd.c b/tools/testing/selftests/bpf/progs/verifier_xadd.c index 05a0a55adb45..7bde3da2f36e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_xadd.c +++ b/tools/testing/selftests/bpf/progs/verifier_xadd.c @@ -121,4 +121,31 @@ l0_%=: r0 = 42; \ " ::: __clobber_all); } +SEC("tc") +__description("xadd with variable stack offset") +__failure +__msg("variable offset stack pointer cannot be passed into helper function") +__msg("Verification failed: Memory Safety: Variable-offset stack access") +__msg("The instruction would access the stack") +__msg("Use a fixed stack offset for the instruction") +__naked void xadd_variable_stack_offset(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r1 = r10; \ + r1 += -16; \ + r1 += r0; \ + r2 = 1; \ + lock *(u64 *)(r1 + 0) += r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; From e9ac05b9827824167f1d8d19df79f8b7808dcc44 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 19:06:37 +0200 Subject: [PATCH 286/660] bpf: Drop dead spill diagnostic condition diag_record_mod() suppresses lineage-preserving register writes when the diagnostic snapshot is unchanged. This is reachable for ALU operations that do not change the represented type or bounds, avoiding a redundant history event while retaining the register's earlier lineage. The condition appears to extend this suppression to spill events, but diag_mod_keeps_lineage() only accepts register writes. The spill producer also supplies a stack target and an explicit origin, so a spill can never satisfy the conjunction. Restrict the condition to writes to reflect the filter's actual contract and avoid suggesting that it applies to spills. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/48e6f021b89562f68850fe21ef8c78719819b04cf9c4e4f50bc791937d37ace8@mail.kernel.org/ Link: https://patch.msgid.link/20260924170646.2366016-4-memxor@gmail.com --- kernel/bpf/diagnostics.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index af97aea49075..a1bd62e4d093 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1567,8 +1567,7 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { event.mod.origin_valid = true; } - if (old_reg && new_reg && - (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE && !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && !event.mod.origin_valid && diag_mod_keeps_lineage(env, &event)) From dac3b7a30d7e5384a61ee46f476dc2c45ceb9453 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 24 Sep 2026 19:06:38 +0200 Subject: [PATCH 287/660] bpf: Report non-sleepable kfunc programs accurately A sleepable kfunc call can fail either because the program is not sleepable or because an otherwise sleepable program has entered a non-sleepable critical section. check_kfunc_call() checks these conditions separately. The first check is only gated by in_sleepable(), so the shared diagnostic can blame an active RCU, preemption-disabled, IRQ-disabled, or locked region even though leaving that region would not make the program sleepable. Adding a second diagnostic entry point only to force the program context would duplicate the API. Reject the call once based on in_sleepable_context(). Teach the shared bpf_diag_ctx_forbidden() reporter and non_sleepable_context_description() to consider active RCU, preempt, IRQ, and lock regions only when the program is sleepable, and to fall back to the non-sleepable program otherwise. A non-sleepable program can still hold that context state, so relying on the existing no-context fallback alone would keep blaming the critical section. Select the suggestion at the kfunc, helper, and global-function sites according to that cause, and choose between the two existing kfunc verifier messages the same way, so a sleepable program is told to leave the critical section and a non-sleepable program is told to become sleepable. This avoids a diagnostic-only wrapper while retaining context history for sleepable programs that enter a forbidden region. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/2e42a1a2bf45f4d2aba7495bdc9f147558055740e2f3c8b9dae255f6c57fc13c@mail.kernel.org/ Link: https://patch.msgid.link/20260924170646.2366016-5-memxor@gmail.com --- kernel/bpf/diagnostics.c | 21 +++++++------- kernel/bpf/verifier.c | 62 ++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index a1bd62e4d093..857b668212fb 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1119,16 +1119,17 @@ void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, const char *constraint, *context; u32 depth; - if (env->cur_state->active_rcu_locks) - ctx_kind = BPF_DIAG_CONTEXT_RCU; - else if (env->cur_state->active_preempt_locks) - ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; - else if (env->cur_state->active_irq_id) - ctx_kind = BPF_DIAG_CONTEXT_IRQ; - else if (env->cur_state->active_locks) - ctx_kind = BPF_DIAG_CONTEXT_LOCK; - else - ctx_kind = BPF_DIAG_CONTEXT_NONE; + ctx_kind = BPF_DIAG_CONTEXT_NONE; + if (env->cur_state->in_sleepable) { + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + } depth = diag_context_depth(env, ctx_kind); opts = (struct bpf_diag_history_opts) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7f220fbcc7e9..ce8c2acb5571 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11122,11 +11122,16 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the call outside the critical section, or use a non-sleepable function."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function."; operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); - bpf_diag_ctx_forbidden(env, *insn_idx, operation, - "Move the call outside the critical section, or use a non-sleepable function."); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion); return -EINVAL; } @@ -12055,14 +12060,16 @@ static inline bool in_sleepable_context(struct bpf_verifier_env *env) static const char *non_sleepable_context_description(struct bpf_verifier_env *env) { - if (env->cur_state->active_rcu_locks) - return "rcu_read_lock region"; - if (env->cur_state->active_preempt_locks) - return "non-preemptible region"; - if (env->cur_state->active_irq_id) - return "IRQ-disabled region"; - if (env->cur_state->active_locks) - return "lock region"; + if (in_sleepable(env)) { + if (env->cur_state->active_rcu_locks) + return "rcu_read_lock region"; + if (env->cur_state->active_preempt_locks) + return "non-preemptible region"; + if (env->cur_state->active_irq_id) + return "IRQ-disabled region"; + if (env->cur_state->active_locks) + return "lock region"; + } return "non-sleepable prog"; } @@ -12154,12 +12161,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } if (fn->might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper."; operation = bpf_diag_fmt(env, "sleepable helper %s#%d", func_id_name(func_id), func_id); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the helper call outside the critical section, or use a non-sleepable helper."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EINVAL; } @@ -14794,11 +14806,20 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } sleepable = bpf_is_kfunc_sleepable(&meta); - if (sleepable && !in_sleepable(env)) { - verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + if (sleepable && !in_sleepable_context(env)) { + const char *suggestion; + + if (in_sleepable(env)) { + verbose(env, "kernel func %s is sleepable within %s\n", + func_name, non_sleepable_context_description(env)); + suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."; + } else { + verbose(env, "program must be sleepable to call sleepable kfunc %s\n", + func_name); + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."; + } operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EACCES; } @@ -14912,15 +14933,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, invalidate_rcu_protected_refs(env); } - if (sleepable && !in_sleepable_context(env)) { - verbose(env, "kernel func %s is sleepable within %s\n", - func_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); - return -EACCES; - } - if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; From 9ed640ee09ab25433f57f4eb2f6dd2e1be0140e0 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Tue, 22 Sep 2026 11:54:40 +0900 Subject: [PATCH 288/660] bpf: Report BPF_F_PREORDER in cgroup program queries Extend direct cgroup program queries to merge (pl->flags & BPF_F_PREORDER) with the cgroup-wide flags when copying prog_attach_flags to user space. Only BPF_F_PREORDER is extracted from pl->flags to ensure transient positioning flags (such as BPF_F_BEFORE, BPF_F_AFTER, BPF_F_ID, or BPF_F_REPLACE) are not leaked to user space. This can introduce new prog_attach_flags[] values with BPF_F_PREORDER set, for example 0x40 (BPF_F_PREORDER), 0x41 (BPF_F_ALLOW_OVERRIDE | BPF_F_PREORDER), or 0x42 (BPF_F_ALLOW_MULTI | BPF_F_PREORDER), depending on the cgroup-wide attachment mode. Older bpftool versions treat such combinations as unknown flags; bpftool support is updated later in this series. Signed-off-by: Hui Su Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922025442.3176057-2-sh_def@163.com --- kernel/bpf/cgroup.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 1cb5e6a6ffc1..2bbe77de89f0 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1515,19 +1515,19 @@ skip_count: id = prog_list_id(pl); if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; + if (prog_attach_flags) { + flags = cgrp->bpf.flags[atype] | + (pl->flags & BPF_F_PREORDER); + if (copy_to_user(prog_attach_flags + i, + &flags, sizeof(flags))) + return -EFAULT; + } if (++i == cnt) break; } - if (prog_attach_flags) { - flags = cgrp->bpf.flags[atype]; - - for (i = 0; i < cnt; i++) - if (copy_to_user(prog_attach_flags + i, - &flags, sizeof(flags))) - return -EFAULT; + if (prog_attach_flags) prog_attach_flags += cnt; - } } prog_ids += cnt; From 47211d1200f0875102ed43b631242a1293938863 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Tue, 22 Sep 2026 11:54:41 +0900 Subject: [PATCH 289/660] selftests/bpf: Test querying BPF_F_PREORDER cgroup attachments Extend cgroup_mprog_opts selftests to verify that BPF_PROG_QUERY reports BPF_F_PREORDER in prog_attach_flags for both direct program and link attachments. Verify that: - Programs attached with BPF_F_PREORDER report BPF_F_ALLOW_MULTI | BPF_F_PREORDER (0x42) in prog_attach_flags. - Programs attached without BPF_F_PREORDER report BPF_F_ALLOW_MULTI (0x2). - Transient flags such as BPF_F_AFTER or BPF_F_LINK are not present in prog_attach_flags. Signed-off-by: Hui Su Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922025442.3176057-3-sh_def@163.com --- .../bpf/prog_tests/cgroup_mprog_opts.c | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c index bb60704a3ef9..9ec5e4f96312 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c @@ -273,11 +273,64 @@ cleanup: close(cg); } +static int find_prog_attach_flags(const struct bpf_prog_query_opts *opts, + __u32 prog_id, __u32 *flags) +{ + __u32 i; + + for (i = 0; i < opts->count; i++) { + if (opts->prog_ids[i] == prog_id) { + *flags = opts->prog_attach_flags[i]; + return 0; + } + } + return -ENOENT; +} + +static void assert_preorder_query_flags(int cg, int atype, + __u32 id1, __u32 id2, + __u32 id3, __u32 id4) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 prog_attach_flags[10] = {}; + __u32 prog_ids[10] = {}; + __u32 flags; + int err; + + optq.prog_ids = prog_ids; + optq.prog_attach_flags = prog_attach_flags; + optq.count = 10; + err = bpf_prog_query_opts(cg, atype, &optq); + if (!ASSERT_OK(err, "prog_query")) + return; + + ASSERT_EQ(optq.count, 4, "count"); + + /* Match by prog_id to avoid relying on query order. */ + err = find_prog_attach_flags(&optq, id1, &flags); + if (ASSERT_OK(err, "find id1")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id1"); + + err = find_prog_attach_flags(&optq, id2, &flags); + if (ASSERT_OK(err, "find id2")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id2"); + + err = find_prog_attach_flags(&optq, id3, &flags); + if (ASSERT_OK(err, "find id3")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id3"); + + err = find_prog_attach_flags(&optq, id4, &flags); + if (ASSERT_OK(err, "find id4")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id4"); +} + static void test_preorder_prog_attach_detach(int atype) { LIBBPF_OPTS(bpf_prog_attach_opts, opta); LIBBPF_OPTS(bpf_prog_detach_opts, optd); - __u32 fd1, fd2, fd3, fd4; + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; struct cgroup_mprog *skel; int cg, err; @@ -294,6 +347,11 @@ static void test_preorder_prog_attach_detach(int atype) fd3 = bpf_program__fd(skel->progs.getsockopt_3); fd4 = bpf_program__fd(skel->progs.getsockopt_4); + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -357,6 +415,8 @@ static void test_preorder_prog_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + err = bpf_prog_detach_opts(fd4, cg, atype, &optd); ASSERT_OK(err, "prog_detach"); assert_mprog_count(cg, atype, 3); @@ -386,7 +446,7 @@ static void test_preorder_link_attach_detach(int atype) LIBBPF_OPTS(bpf_cgroup_opts, opta); struct bpf_link *link1, *link2, *link3, *link4; struct cgroup_mprog *skel; - __u32 fd2; + __u32 fd2, id1, id2, id3, id4; int cg; cg = test__join_cgroup("/preorder_link_attach_detach"); @@ -399,6 +459,11 @@ static void test_preorder_link_attach_detach(int atype) fd2 = bpf_program__fd(skel->progs.getsockopt_2); + id1 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_1)); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_3)); + id4 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_4)); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -460,6 +525,8 @@ static void test_preorder_link_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + bpf_link__destroy(link4); assert_mprog_count(cg, atype, 3); From c39bf6b6705f227ebdd191453122ac3db956db0e Mon Sep 17 00:00:00 2001 From: Hui Su Date: Tue, 22 Sep 2026 11:54:42 +0900 Subject: [PATCH 290/660] bpftool: Add support for BPF_F_PREORDER cgroup attach flag Commit 4b82b181a26c ("bpf: Allow pre-ordering for bpf cgroup progs") introduced BPF_F_PREORDER to request pre-order execution across the cgroup hierarchy. Furthermore, attachments legitimately use combinations such as BPF_F_ALLOW_MULTI | BPF_F_PREORDER or BPF_F_ALLOW_OVERRIDE | BPF_F_PREORDER. With BPF_PROG_QUERY reporting the per-program BPF_F_PREORDER attribute, bpftool's exact-match formatter falls back to "unknown(40)" when BPF_F_PREORDER is present alone, or "unknown(41)" / "unknown(42)" when combined with BPF_F_ALLOW_OVERRIDE or BPF_F_ALLOW_MULTI. Additionally, do_attach() only accepts "multi" and "override", rejecting "preorder" with "unknown option". Before: $ bpftool cgroup show 1234 cgroup_inet_ingress unknown(42) test_prog $ bpftool cgroup attach cgroup_inet_ingress id 5678 multi preorder Error: unknown option: preorder After: $ bpftool cgroup show 1234 cgroup_inet_ingress multi,preorder test_prog $ bpftool cgroup attach cgroup_inet_ingress id 5678 multi preorder (attaches successfully) Refactor the attach flags formatter into a bitmask formatter that outputs comma-separated flag names for plain text while preserving unrecognized bits as "unknown(...)". Render attach flags as an array of flag names in JSON output. Accept "preorder" in do_attach(), update the cgroup documentation and synopsis to express valid flag combinations, and teach bash completion about them ("multi" or "override" optionally combined with "preorder"). Signed-off-by: Hui Su Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20260922025442.3176057-4-sh_def@163.com --- .../bpftool/Documentation/bpftool-cgroup.rst | 25 +++-- tools/bpf/bpftool/bash-completion/bpftool | 4 +- tools/bpf/bpftool/cgroup.c | 99 +++++++++++++------ 3 files changed, 89 insertions(+), 39 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst index e8185596a759..d1b8193dc576 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst @@ -45,7 +45,7 @@ CGROUP COMMANDS | **cgroup_unix_recvmsg** | **cgroup_sysctl** | | **cgroup_getsockopt** | **cgroup_setsockopt** | | **cgroup_inet_sock_release** } -| *ATTACH_FLAGS* := { **multi** | **override** } +| *ATTACH_FLAGS* := { [ **multi** | **override** ] [ **preorder** ] } DESCRIPTION =========== @@ -75,20 +75,27 @@ bpftool cgroup attach *CGROUP* *ATTACH_TYPE* *PROG* [*ATTACH_FLAGS*] Attach program *PROG* to the cgroup *CGROUP* with attach type *ATTACH_TYPE* and optional *ATTACH_FLAGS*. - *ATTACH_FLAGS* can be one of: **override** if a sub-cgroup installs some + *ATTACH_FLAGS* can include: **override** if a sub-cgroup installs some bpf program, the program in this cgroup yields to sub-cgroup program; **multi** if a sub-cgroup installs some bpf program, that cgroup program - gets run in addition to the program in this cgroup. + gets run in addition to the program in this cgroup; + **preorder** requests ancestor-to-descendant execution for this program, + before non-preorder programs, which execute descendants-to-ancestors + across the cgroup hierarchy. Note that **preorder** alone does not enable + multi-program attachment; specify **multi** together with **preorder** to + attach multiple programs. - Only one program is allowed to be attached to a cgroup with no attach flags - or the **override** flag. Attaching another program will release old - program and attach the new one. + Only one program is allowed to be attached to a cgroup unless the + **multi** flag is specified. Without **multi**, attaching another program + replaces the existing program, provided the **override** setting matches. Multiple programs are allowed to be attached to a cgroup with **multi**. - They are executed in FIFO order (those that were attached first, run - first). + Programs marked with **preorder** are placed before non-preorder programs + in the effective program array. Within each ordering class at the same + cgroup level, attachment order is preserved. - Non-default *ATTACH_FLAGS* are supported by kernel version 4.14 and later. + **multi** and **override** are supported by kernel version 4.14 and later. + **preorder** was introduced upstream in Linux 6.15. *ATTACH_TYPE* can be one of: diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index c9e8761e4ef2..9d9ced270685 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -1064,7 +1064,6 @@ _bpftool() attach|detach) local BPFTOOL_CGROUP_ATTACH_TYPES="$(bpftool feature list_builtins attach_types 2>/dev/null | \ grep '^cgroup_')" - local ATTACH_FLAGS='multi override' # Check for $prev = $command first if [ $prev = $command ]; then _filedir @@ -1094,7 +1093,8 @@ _bpftool() # "id|pinned|tag|name" (we already checked for # that). This should only leave the case when # we need attach flags for "attach" commamnd. - _bpftool_one_of_list "$ATTACH_FLAGS" + _bpftool_one_of_list 'multi override' + _bpftool_once_attr 'preorder' fi return 0 ;; diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index ce69d1e5468e..fee1a260f6d1 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c @@ -56,7 +56,7 @@ static const int cgroup_attach_types[] = { }; #define HELP_SPEC_ATTACH_FLAGS \ - "ATTACH_FLAGS := { multi | override }" + "ATTACH_FLAGS := { [ multi | override ] [ preorder ] }" #define HELP_SPEC_ATTACH_TYPES \ " ATTACH_TYPE := { cgroup_inet_ingress | cgroup_inet_egress |\n" \ @@ -138,11 +138,69 @@ out: close(fd); } +static const struct { + __u32 flag; + const char *name; +} attach_flag_names[] = { + { BPF_F_ALLOW_MULTI, "multi" }, + { BPF_F_ALLOW_OVERRIDE, "override" }, + { BPF_F_PREORDER, "preorder" }, +}; + +static const char *format_attach_flags(__u32 flags, char *buf, size_t sz) +{ + size_t len = 0; + size_t i; + int n; + + buf[0] = '\0'; + for (i = 0; i < ARRAY_SIZE(attach_flag_names); i++) { + if (flags & attach_flag_names[i].flag) { + n = snprintf(buf + len, sz - len, "%s%s", + len ? "," : "", attach_flag_names[i].name); + if (n < 0 || (size_t)n >= sz - len) + return buf; + len += n; + flags &= ~attach_flag_names[i].flag; + } + } + + if (flags) + snprintf(buf + len, sz - len, "%sunknown(%x)", + len ? "," : "", flags); + + return buf; +} + +static void show_attach_flags_json(__u32 flags) +{ + char buf[32]; + size_t i; + + jsonw_name(json_wtr, "attach_flags"); + jsonw_start_array(json_wtr); + + for (i = 0; i < ARRAY_SIZE(attach_flag_names); i++) { + if (!(flags & attach_flag_names[i].flag)) + continue; + + jsonw_string(json_wtr, attach_flag_names[i].name); + flags &= ~attach_flag_names[i].flag; + } + + if (flags) { + snprintf(buf, sizeof(buf), "unknown(%x)", flags); + jsonw_string(json_wtr, buf); + } + + jsonw_end_array(json_wtr); +} + static int show_bpf_prog(int id, enum bpf_attach_type attach_type, - const char *attach_flags_str, - int level) + __u32 attach_flags, int level) { char prog_name[MAX_PROG_FULL_NAME]; + char attach_flags_str[64]; const char *attach_btf_name = NULL; struct bpf_prog_info info = {}; const char *attach_type_str; @@ -182,7 +240,7 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, else jsonw_uint_field(json_wtr, "attach_type", attach_type); if (!(query_flags & BPF_F_QUERY_EFFECTIVE)) - jsonw_string_field(json_wtr, "attach_flags", attach_flags_str); + show_attach_flags_json(attach_flags); jsonw_string_field(json_wtr, "name", prog_name); if (attach_btf_name) jsonw_string_field(json_wtr, "attach_btf_name", attach_btf_name); @@ -198,7 +256,9 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, if (query_flags & BPF_F_QUERY_EFFECTIVE) printf(" %-15s", prog_name); else - printf(" %-15s %-15s", attach_flags_str, prog_name); + printf(" %-17s %-15s", + format_attach_flags(attach_flags, attach_flags_str, + sizeof(attach_flags_str)), prog_name); if (attach_btf_name) printf(" %-15s", attach_btf_name); else if (info.attach_btf_id) @@ -264,7 +324,7 @@ static int show_effective_bpf_progs(int cgroup_fd, enum bpf_attach_type type, return 0; for (iter = 0; iter < p.prog_cnt; iter++) - show_bpf_prog(prog_ids[iter], type, NULL, level); + show_bpf_prog(prog_ids[iter], type, 0, level); return 0; } @@ -274,9 +334,7 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, { LIBBPF_OPTS(bpf_prog_query_opts, p); __u32 prog_attach_flags[1024] = {0}; - const char *attach_flags_str; __u32 prog_ids[1024] = {0}; - char buf[32]; __u32 iter; int ret; @@ -296,24 +354,7 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, __u32 attach_flags; attach_flags = prog_attach_flags[iter] ?: p.attach_flags; - - switch (attach_flags) { - case BPF_F_ALLOW_MULTI: - attach_flags_str = "multi"; - break; - case BPF_F_ALLOW_OVERRIDE: - attach_flags_str = "override"; - break; - case 0: - attach_flags_str = ""; - break; - default: - snprintf(buf, sizeof(buf), "unknown(%x)", attach_flags); - attach_flags_str = buf; - } - - show_bpf_prog(prog_ids[iter], type, - attach_flags_str, level); + show_bpf_prog(prog_ids[iter], type, attach_flags, level); } return 0; @@ -377,7 +418,7 @@ static int do_show(int argc, char **argv) else if (query_flags & BPF_F_QUERY_EFFECTIVE) printf("%-8s %-15s %-15s\n", "ID", "AttachType", "Name"); else - printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType", + printf("%-8s %-15s %-17s %-15s\n", "ID", "AttachType", "AttachFlags", "Name"); btf_vmlinux = libbpf_find_kernel_btf(); @@ -531,7 +572,7 @@ static int do_show_tree(int argc, char **argv) "ID", "AttachType", "Name"); else printf("%s\n" - "%-8s %-15s %-15s %-15s\n", + "%-8s %-15s %-17s %-15s\n", "CgroupPath", "ID", "AttachType", "AttachFlags", "Name"); @@ -593,6 +634,8 @@ static int do_attach(int argc, char **argv) attach_flags |= BPF_F_ALLOW_MULTI; } else if (is_prefix(argv[i], "override")) { attach_flags |= BPF_F_ALLOW_OVERRIDE; + } else if (is_prefix(argv[i], "preorder")) { + attach_flags |= BPF_F_PREORDER; } else { p_err("unknown option: %s", argv[i]); goto exit_cgroup; From e55220162aae99622d9e80d4700e51c13a6f2a16 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Thu, 24 Sep 2026 09:26:06 -0700 Subject: [PATCH 291/660] bpf: Look at frame 0 when telling async callback entries apart push_async_cb() sets in_async_callback_fn and async_entry_cnt on the callback's own frame, which is frame 0 of the fresh state it starts, and setup_func_entry() copies neither, so a subprog called by the callback carries neither. Two places read them from the innermost frame instead. is_state_visited() skips the infinite loop check when two states differ in async_entry_cnt, because seeing the same state on a second entry into an async callback is not a loop. A callback which re-arms itself and calls a subprog therefore compares the two entries at a loop inside that subprog, where the innermost frame is the subprog's and has no flag set, and the state is rejected: infinite loop detected at insn 60 push_callback_call() numbers a new entry as the caller's count plus one. When the callback re-arms itself from a subprog, the caller is that subprog's frame with a count of 0, so every entry is numbered 1, the check above never sees a difference, and a loop anywhere in such a callback is rejected the same way. Read the flag and the count from frame 0 in both places. A loop within a single entry still has a matching count and is still caught, and frame 0 of a non-async state does not have the flag set, so nothing else changes. The two other readers are already correct: check_return_code() reads frame[0], and the one in do_check() is reached only after an early return for curframe != 0. Fixes: bfc6bb74e4f1 ("bpf: Implement verifier support for validation of async callbacks.") Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924162609.1746610-2-puranjay@kernel.org --- kernel/bpf/states.c | 4 ++-- kernel/bpf/verifier.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 5c7169bfb4e9..c1fbb339329f 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1280,10 +1280,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) continue; if (sl->state.branches) { - struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; + struct bpf_func_state *frame = sl->state.frame[0]; if (frame->in_async_callback_fn && - frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { + frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { /* Different async_entry_cnt means that the verifier is * processing another entry into async callback. * Seeing the same state is not an indication of infinite diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ce8c2acb5571..200ec0f71617 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10998,7 +10998,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins if (IS_ERR(async_cb)) return PTR_ERR(async_cb); callee = async_cb->frame[0]; - callee->async_entry_cnt = caller->async_entry_cnt + 1; + callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; /* Convert bpf_timer_set_callback() args into timer callback args */ err = set_callee_state_cb(env, caller, callee, insn_idx); From 84bff37db2ad26be7d707cd9030ad5521bf16d9a Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Thu, 24 Sep 2026 09:26:07 -0700 Subject: [PATCH 292/660] selftests/bpf: Add timer tests for a re-arming callback with a loop Existing timer callbacks either do not re-arm through bpf_timer_set_callback(), which is what starts another async callback entry, or do not reach a loop once they do. Cover that: loop_cb() re-arms itself and reaches a bounded loop through sum_to(), which is static and not inlined, so the loop is walked in a frame below the callback's rather than in a separately verified global subprog. The re-arm goes through rearm(), also static and not inlined, so the caller frame at bpf_timer_set_callback() is not the callback's own frame and carries no async_entry_cnt of its own. That covers both frames the preceding fix corrects; without it the program is rejected with "infinite loop detected". Telling async callback entries apart must not stop the verifier catching a real loop in a callback, and no existing test covers that either, so also check that a callback which never returns is still rejected. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924162609.1746610-3-puranjay@kernel.org --- .../testing/selftests/bpf/prog_tests/timer.c | 33 ++++++++++ tools/testing/selftests/bpf/progs/timer.c | 60 ++++++++++++++++++- .../selftests/bpf/progs/timer_failure.c | 29 +++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/timer.c b/tools/testing/selftests/bpf/prog_tests/timer.c index 09ff21e1ad2f..178223190b8b 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer.c +++ b/tools/testing/selftests/bpf/prog_tests/timer.c @@ -262,6 +262,34 @@ static int timer_cancel_async(struct timer *timer_skel) return 0; } +/* + * A timer callback which re-arms itself and reaches a loop through a call: + * the two entries into the callback meet at the loop, in a frame of its own. + */ +static int timer_loop_rearm(struct timer *timer_skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd, i; + + err = timer__attach(timer_skel); + if (!ASSERT_OK(err, "timer_attach")) + return err; + + timer_skel->bss->loop_rearm = 1; + + prog_fd = bpf_program__fd(timer_skel->progs.test_loop_rearm); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run")) + return err; + + for (i = 0; i < 100 && timer_skel->bss->loop_sum < 120 * 2; i++) + usleep(1000); + + timer__detach(timer_skel); + ASSERT_EQ(timer_skel->bss->loop_sum, 120 * 2, "loop_sum"); + return 0; +} + static void test_timer(int (*timer_test_fn)(struct timer *timer_skel)) { struct timer *timer_skel = NULL; @@ -287,6 +315,11 @@ void serial_test_timer(void) RUN_TESTS(timer_failure); } +void serial_test_timer_loop_rearm(void) +{ + test_timer(timer_loop_rearm); +} + void serial_test_timer_stress(void) { test_timer(timer_stress); diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c index d6d5fefcd9b1..aa966dbed02e 100644 --- a/tools/testing/selftests/bpf/progs/timer.c +++ b/tools/testing/selftests/bpf/progs/timer.c @@ -6,6 +6,7 @@ #include #include #include +#include "bpf_experimental.h" #define CLOCK_MONOTONIC 1 #define CLOCK_BOOTTIME 7 @@ -57,9 +58,12 @@ struct { __type(key, int); __type(value, struct elem); } abs_timer SEC(".maps"), soft_timer_pinned SEC(".maps"), abs_timer_pinned SEC(".maps"), - race_array SEC(".maps"); + race_array SEC(".maps"), loop_array SEC(".maps"); __u64 bss_data; +__u64 loop_sum; +int loop_rearm; /* number of times loop_cb() re-arms itself */ +__u64 zero; __u64 abs_data; __u64 err; __u64 ok; @@ -139,6 +143,60 @@ static int timer_cb1(void *map, int *key, struct bpf_timer *timer) return 0; } +/* + * Static and not inlined, so the loop is walked in a frame below the + * callback's rather than in a separately verified global subprog. + */ +static __noinline int sum_to(__u64 n) +{ + __u64 i, sum = 0; + + for (i = zero; i < n && can_loop; i++) + sum += i; + + return sum; +} + +static int loop_cb(void *map, int *key, struct elem *val); + +/* + * Re-arms from a frame below the callback's, where the caller frame carries + * no async_entry_cnt of its own. + */ +static __noinline void rearm(struct elem *val) +{ + if (loop_rearm > 0) { + loop_rearm--; + /* set_callback is what starts another async callback entry */ + bpf_timer_set_callback(&val->t, loop_cb); + bpf_timer_start(&val->t, 0, 0); + } +} + +/* Re-arms itself and reaches a bounded loop through a call. */ +static int loop_cb(void *map, int *key, struct elem *val) +{ + loop_sum += sum_to(16); + rearm(val); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG2(test_loop_rearm, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&loop_array, &key); + if (!timer) + return 0; + + bpf_timer_init(timer, &loop_array, CLOCK_MONOTONIC); + bpf_timer_set_callback(timer, loop_cb); + bpf_timer_start(timer, 0, 0); + return 0; +} + SEC("fentry/bpf_fentry_test1") int BPF_PROG2(test1, int, a) { diff --git a/tools/testing/selftests/bpf/progs/timer_failure.c b/tools/testing/selftests/bpf/progs/timer_failure.c index 5a2e9dabf1c6..0538269101ca 100644 --- a/tools/testing/selftests/bpf/progs/timer_failure.c +++ b/tools/testing/selftests/bpf/progs/timer_failure.c @@ -66,3 +66,32 @@ long BPF_PROG2(test_bad_ret, int, a) return 0; } + +/* + * A real loop inside an async callback must still be rejected: both entries + * into the callback have the same async_entry_cnt, so telling entries apart + * does not apply here. + */ +static int timer_cb_infinite_loop(void *map, int *key, struct elem *val) +{ + for (;;) {} + + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__failure __msg("infinite loop detected") +long BPF_PROG2(test_infinite_loop_cb, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&timer_map, &key); + if (timer) { + bpf_timer_init(timer, &timer_map, CLOCK_BOOTTIME); + bpf_timer_set_callback(timer, timer_cb_infinite_loop); + bpf_timer_start(timer, 1000, 0); + } + + return 0; +} From 31575dcba0ec9319a218d9f5ff46b6a2be82c340 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 18:25:33 +0000 Subject: [PATCH 293/660] selftests/bpf: Run callx prog_tests once prog_tests/tests.h is generated from 'void test_*(void)' definitions. callx_func_ptr_map.c and callx_rodata_lskel.c define the test twice, under #if and under #else, so both tests are registered and run twice: DEFINE_TEST(callx_func_ptr_map) DEFINE_TEST(callx_func_ptr_map) DEFINE_TEST(callx_rodata_lskel) DEFINE_TEST(callx_rodata_lskel) Make both definitions static and call them from a single test function. Fixes: b018e998e645 ("selftests/bpf: Add tests for callx through pointers in read-only data") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924182533.281399-1-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/callx_func_ptr_map.c | 9 +++++++-- .../selftests/bpf/prog_tests/callx_rodata_lskel.c | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c index b58ebee9b1ef..42b33d5876b1 100644 --- a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c @@ -100,7 +100,7 @@ static void run(int prog_fd, int expected, const char *name) ASSERT_EQ(topts.retval, expected, name); } -void test_callx_func_ptr_map(void) +static void __test_callx_func_ptr_map(void) { int int_id, proto_id, map_fd = -1, prog_fd = -1, reader_fd = -1, fd, zero = 0; int main_id, add1_id; @@ -194,9 +194,14 @@ out: #else -void test_callx_func_ptr_map(void) +static void __test_callx_func_ptr_map(void) { test__skip(); } #endif + +void test_callx_func_ptr_map(void) +{ + __test_callx_func_ptr_map(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c index 6a43b880d730..fc5181c5b5ba 100644 --- a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -21,7 +21,7 @@ static void run(int prog_fd, int expected, const char *name) * of its functions. The copies have what user space puts into .rodata before * the load. The program that doesn't have callx uses .rodata. */ -void test_callx_rodata_lskel(void) +static void __test_callx_rodata_lskel(void) { struct callx_rodata_lskel *skel; @@ -61,9 +61,14 @@ out: #else -void test_callx_rodata_lskel(void) +static void __test_callx_rodata_lskel(void) { test__skip(); } #endif + +void test_callx_rodata_lskel(void) +{ + __test_callx_rodata_lskel(); +} From 93df8ae3267f19bf0c1135d26a0e21145dd0ea1c Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 23 Sep 2026 14:42:16 +0000 Subject: [PATCH 294/660] bpf: Adjust bpf_func to account for CFI offset in bpf_jit_free on riscv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When CFI is enabled, the actual BPF program entry point is shifted forward by a CFI preamble. During bpf_jit_free(), this shifted pointer causes the wrong ro_header to be fetched, leading to a potential invalid memory free. Fix this by subtracting cfi_get_offset() from prog->bpf_func to correctly restore the original JITed allocation address before freeing. Fixes: e63985ecd226 ("bpf, riscv64/cfi: Support kCFI + BPF on riscv64") Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://patch.msgid.link/20260923144216.934701-1-pulehui@huaweicloud.com --- arch/riscv/net/bpf_jit_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 2fb0b4e198b9..470a6ace5662 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -244,6 +244,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->ctx.offset); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); From bb83425b3cdcb7bb3c629598a4d5efad9cc80733 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:23 +0000 Subject: [PATCH 295/660] bpf, x86: Fix timed may_goto with private stack arch_bpf_timed_may_goto() computes the address of count and timestamp as rbp + offset. When the prog uses private stack the JIT maps r10 to r9 and nothing is reserved under rbp, so bpf_check_timed_may_goto() reads and writes the timestamp in the kernel stack, where arch_bpf_timed_may_goto() has just saved r0-r5 or further down. The following loop in fentry prog with r0-r5 set to zero: 1: may_goto +2 r6 += 1 goto 1b leaves r3 with ktime in it when the prog has 64 bytes of stack and ends after 65535 iterations when it has 128. Add the frame pointer to the offset in the JIT, where rbp vs r9 is known, and pass the pointer in AX. arm64 and powerpc JITs keep the address of private stack in BPF_REG_FP and are not affected. Fixes: 2fb761823ead ("bpf, x86: Add x86 JIT support for timed may_goto") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-2-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 12 ++++++++++++ arch/x86/net/bpf_timed_may_goto.S | 10 ++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e2e531dd1e0b..6c7a0578760e 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2961,6 +2961,18 @@ populate_extable: ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - outgoing_rsp, &prog); } + if (func == (u8 *)arch_bpf_timed_may_goto) { + u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; + + /* + * AX has the offset of count and timestamp + * in the stack. Turn it into a pointer. + * add r10, rbp or add r10, r9 + */ + maybe_emit_mod(&prog, BPF_REG_AX, fp, true); + EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); + ip += 3; + } if (priv_frame_ptr) { push_r9(&prog); ip += 2; diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 54c690cae190..6c6e03f725a3 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -11,12 +11,6 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ANNOTATE_NOENDBR - /* - * r10 passes us stack depth, load the pointer to count and timestamp - * into r10 by adding it to BPF frame pointer. - */ - leaq (%rbp, %r10, 1), %r10 - /* Setup frame. */ pushq %rbp movq %rsp, %rbp @@ -30,8 +24,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) pushq %r8 /* - * r10 has the pointer to count and timestamp, pass it as first - * argument. + * r10 has the pointer to count and timestamp. JIT added the frame + * pointer to the offset. Pass it as first argument. */ movq %r10, %rdi From d17107a3051b78b06001be2f9dfffb9b9ec5de8d Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:24 +0000 Subject: [PATCH 296/660] selftests/bpf: Add test for timed may_goto with private stack Run fentry progs with 64 and 128 bytes of stack that loop until may_goto expires. Check that r0-r5 are intact after the loop and that it made more than 65535 iterations. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-3-alexei.starovoitov@gmail.com --- .../bpf/prog_tests/may_goto_priv_stack.c | 41 +++++++++++++++ .../selftests/bpf/progs/may_goto_priv_stack.c | 52 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c create mode 100644 tools/testing/selftests/bpf/progs/may_goto_priv_stack.c diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c new file mode 100644 index 000000000000..4f8476d36250 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "may_goto_priv_stack.skel.h" + +static void check_regs(__u64 *regs) +{ + int i; + + for (i = 0; i < 6; i++) + ASSERT_EQ(regs[i], 0, "reg"); + /* count is refreshed at least once when may_goto is timed */ + ASSERT_GT(regs[6], 0xffff, "iterations"); +} + +void test_may_goto_priv_stack(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct may_goto_priv_stack *skel; + int err; + + skel = may_goto_priv_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + err = may_goto_priv_stack__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto out; + + memset(skel->bss->regs_64, 0xff, sizeof(skel->bss->regs_64)); + memset(skel->bss->regs_128, 0xff, sizeof(skel->bss->regs_128)); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.priv_stack_64), &topts); + if (!ASSERT_OK(err, "test_run")) + goto out; + + check_regs(skel->bss->regs_64); + check_regs(skel->bss->regs_128); +out: + may_goto_priv_stack__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c new file mode 100644 index 000000000000..5c2aa0aa5055 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* r0-r5 after the loop and the number of iterations */ +__u64 regs_64[7]; +__u64 regs_128[7]; + +/* + * fentry prog with 64 or more bytes of stack uses private stack. + * Loop until may_goto expires. r0-r5 should stay zero. + */ +#define TIMED_MAY_GOTO_PRIV_STACK(size, func) \ +SEC("fentry/" #func) \ +__naked void priv_stack_##size(void) \ +{ \ + asm volatile ( \ + "r6 = 0;" \ + "*(u64 *)(r10 - " #size ") = r6;" \ + "r7 = %[regs] ll;" \ + "r0 = 0;" \ + "r1 = 0;" \ + "r2 = 0;" \ + "r3 = 0;" \ + "r4 = 0;" \ + "r5 = 0;" \ +"1:" \ + ".8byte %[may_goto];" \ + "r6 += 1;" \ + "goto 1b;" \ + "*(u64 *)(r7 + 0) = r0;" \ + "*(u64 *)(r7 + 8) = r1;" \ + "*(u64 *)(r7 + 16) = r2;" \ + "*(u64 *)(r7 + 24) = r3;" \ + "*(u64 *)(r7 + 32) = r4;" \ + "*(u64 *)(r7 + 40) = r5;" \ + "*(u64 *)(r7 + 48) = r6;" \ + "r0 = 0;" \ + "exit;" \ + : \ + : [regs]"i"(®s_##size), \ + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) \ + : __clobber_all); \ +} + +TIMED_MAY_GOTO_PRIV_STACK(64, bpf_fentry_test1) +TIMED_MAY_GOTO_PRIV_STACK(128, bpf_fentry_test2) + +char _license[] SEC("license") = "GPL"; From 67a3b916a7ae7c1a6bed08555417d444de6f5838 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:25 +0000 Subject: [PATCH 297/660] bpf: Adjust pc-relative insn copied into its own patch bpf_do_misc_fixups() inits may_goto count by patching the first insn of a subprog with [ST, ST, first insn]. bpf_convert_ctx_accesses() patches insn that needs a barrier with [nospec, insn]. bpf_adj_branches() doesn't adjust insns inside the patch, so when the copy is a call, ld_imm64 of a func or a jump that points backward its target is off by the number of insns in front of it: static int A(void) { return 1; } static int A2(void) { return 2; } static int B(void) { int i, ret = A(); for (i = 0; i < 10 && can_loop; i++) ... return ret; } When A is placed in front of B and may_goto is timed 'call A' becomes call of A + 2. B returns 2 when A + 2 is the start of A2. Otherwise the prog is rejected with "verifier bug: No program to jit at insn". Add bpf_adj_moved_insn() and use it for both copies. Fixes: 011832b97b31 ("bpf: Introduce may_goto instruction") Fixes: d6f1c85f2253 ("bpf: Fall back to nospec for Spectre v1") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-4-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 55 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index a5d335a47d26..ae9713d292df 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -383,6 +383,44 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, return new_prog; } +/* + * insn was moved down by delta insns inside its own patch. Operands relative + * to the pc that point in front of the old position did not move with it. + */ +static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) +{ + u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); + s64 off = insn->imm, off_min = S32_MIN; + bool is_imm = true; + + if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { + /* subprog that started at the old position starts with the patch */ + if (off >= 0) + return 0; + } else if ((class == BPF_JMP || class == BPF_JMP32) && + op != BPF_CALL && op != BPF_EXIT) { + if (insn->code != (BPF_JMP32 | BPF_JA)) { + off = insn->off; + off_min = S16_MIN; + is_imm = false; + } + /* jump to itself stays */ + if (off >= -1) + return 0; + } else { + return 0; + } + + off -= delta; + if (off < off_min) + return -ERANGE; + if (is_imm) + insn->imm = off; + else + insn->off = off; + return 0; +} + /* * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the * jump offset by 'delta'. @@ -905,7 +943,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) struct bpf_insn *patch = insn_buf; *patch++ = BPF_ST_NOSPEC(); - *patch++ = *insn; + *patch = *insn; + ret = bpf_adj_moved_insn(patch++, 1); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[i + delta].orig_idx); + return ret; + } cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2597,7 +2641,14 @@ next_insn: BPF_MAX_LOOPS); } /* Copy first actual insn to preserve it */ - insn_buf[cnt++] = env->prog->insnsi[subprog_start]; + insn_buf[cnt] = env->prog->insnsi[subprog_start]; + ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[subprog_start].orig_idx); + return ret; + } + cnt++; new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); if (!new_prog) From 913a5466dbfc28c82ebe8dd85fec0ddc1407a362 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:26 +0000 Subject: [PATCH 298/660] selftests/bpf: Add tests for pc-relative insn copied into its own patch Add tests for may_goto in a subprog that starts with a call or with ld_imm64 of a func located in front of it, and for nospec inserted in front of such call. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-5-alexei.starovoitov@gmail.com --- .../selftests/bpf/progs/verifier_may_goto_1.c | 121 ++++++++++++++++++ .../selftests/bpf/progs/verifier_unpriv.c | 37 ++++++ 2 files changed, 158 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index db7e30da234f..faf9eec0a63e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -173,4 +173,125 @@ __naked void timed_may_goto_preserves_regs(void) : __clobber_all); } +__used __naked static void mg_ret1(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_ret2(void) +{ + asm volatile ( + "r0 = 2;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_ret1 and mg_ret2 are placed in front of mg_call_first, + * so the offset of its call is negative. + */ +SEC("socket") +__description("may_goto in subprog that starts with a call") +__success +__retval(1) +__naked void may_goto_subprog_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "call mg_ret2;" + "call mg_call_first;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_stop(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_cont(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* r1 is zero, but not a constant, so that bpf_loop() is not inlined */ +__used __naked static void mg_func_first(void) +{ + asm volatile ( + "r2 = %[mg_cb_stop] ll;" + "r4 = r1;" + "r1 = 2;" + "r3 = 0;" + "call %[bpf_loop];" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm(bpf_loop), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_cb_stop stops bpf_loop() after the first iteration, + * mg_cb_cont lets it do both. + */ +SEC("socket") +__description("may_goto in subprog that starts with ld_imm64 of a func") +__success +__retval(1) +__naked void may_goto_subprog_func_first(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = %[mg_cb_stop] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = %[mg_cb_cont] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "call %[bpf_ktime_get_ns];" + "r1 = r0;" + "r1 >>= 63;" + "call mg_func_first;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm_addr(mg_cb_cont), + __imm(bpf_loop), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 3069e70fbcbd..a54b5a58b945 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -1035,4 +1035,41 @@ __naked void stack_write_nospec_slot_index(void) " ::: __clobber_all); } +__noinline int nospec_global_func(int x) +{ + return x + 1; +} + +__used __naked static void nospec_call_subprog(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r1 = 0; \ + if r0 < 0x100 goto l0_%=; \ + /* executed only speculatively */ \ + r1 = r10; \ +l0_%=: call nospec_global_func; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* nospec_global_func is placed in front of nospec_call_subprog */ +SEC("socket") +__description("unpriv: nospec in front of a call") +__success __success_unpriv +__caps_unpriv(CAP_BPF) +__retval(1) +__naked void nospec_backward_call(void) +{ + asm volatile (" \ + r1 = 0; \ + call nospec_global_func; \ + call nospec_call_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 8a12a00f6c5dba70a6d908fe27a3c980d42b9515 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:27 +0000 Subject: [PATCH 299/660] bpf: Fix overflow of jump offset in may_goto expansion may_goto is expanded into 'if AX == 0 goto off + 5' when it's timed, off + 2 when it's not and off - 1 when it jumps backward. The sum is stored into 16-bit off without a range check, so may_goto +32763 turns into: if r12 == 0x0 goto pc-32768 and the prog jumps backward when may_goto expires. Emit 'if AX != 0 goto +1; gotol' when the offset doesn't fit and compute the offset from the number of insns in the expansion. JITs that don't support gotol will reject such prog. Fixes: 011832b97b31 ("bpf: Introduce may_goto instruction") Fixes: e723608bf428 ("bpf: Add verifier support for timed may_goto") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-6-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 68 +++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index ae9713d292df..e9c2d6c06218 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1690,6 +1690,30 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat return 0; } +/* + * Expand may_goto: load the count from the stack, jump to the target of + * may_goto when it is zero, then the tail that updates the count. + * Use gotol when the target is too far for 16-bit offset of a conditional jump. + */ +static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, + const struct bpf_insn *tail, int tail_cnt) +{ + int cnt = 0; + + /* Forward jump has to step over the tail */ + off = off >= 0 ? off + tail_cnt : off - 1; + + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); + return cnt + tail_cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1967,6 +1991,18 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { int stack_off_cnt = -stack_depth - 16; + /* + * AX is used as an argument to pass in stack_off_cnt + * (to add to r10/fp), and also as the return value of + * the call to arch_bpf_timed_may_goto. + */ + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), + BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), + BPF_EMIT_CALL(arch_bpf_timed_may_goto), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), + }; /* * Two 8 byte slots, depth-16 stores the count, and @@ -1983,22 +2019,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * after subtraction, rinse and repeat. */ stack_depth_extra = 16; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); - insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); - insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); - cnt = 7; + cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2010,16 +2032,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8; + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), + }; stack_depth_extra = 8; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); - cnt = 4; + cnt = may_goto_expand(insn_buf, insn->off, stack_off, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) From 948c658c93decb0cf88e4d6033db6cf3847a9a85 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:28 +0000 Subject: [PATCH 300/660] selftests/bpf: Add tests for may_goto with far target Generate syscall progs with may_goto that jumps up to 32767 insns forward and 32768 insns backward. The prog loops until may_goto expires and returns 0 when it lands on the target. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-7-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/may_goto_far.c | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/may_goto_far.c diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c new file mode 100644 index 000000000000..8d0ff30bd068 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#define MAY_GOTO(off) BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, off, 0) +#define FILL_CNT 33000 + +static struct bpf_insn *fill(struct bpf_insn *insn, int cnt, int reg) +{ + while (cnt--) + *insn++ = BPF_MOV64_REG(BPF_REG_0, reg); + return insn; +} + +/* + * Syscall prog that loops until may_goto expires. Its target is 'off' insns + * away: + * + * r6 = r7 = 0, but not a constant for the verifier + * r8 = 2 + * r9 = 0 + * r0 = r8 x FILL_CNT or 1 + * 1: if r6 == r9 goto +1 + * exit + * r6 = r8 + * r0 = r9 x N when off is negative + * if r7 != r9 goto 3f + * 2: may_goto off (1b or 4f) + * goto 2b + * 3: r0 = r8 x N when off is positive + * 4: exit + * + * The prog returns 0 when may_goto jumps to its target and 2 when + * it lands in one of 'r0 = r8' areas. There are no constants between + * may_goto and its target to keep the distance when constants are blinded. + */ +static void test_far(int off) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, .prog_flags = BPF_F_SLEEPABLE); + LIBBPF_OPTS(bpf_test_run_opts, topts); + int front = off < 0 ? 1 : FILL_CNT; + int mid = off < 0 ? -off - 5 : 1; + int back = off < 0 ? FILL_CNT : off - 1; + struct bpf_insn *insns, *insn; + int fd, err; + + insns = calloc(front + mid + back + 16, sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "calloc")) + return; + + insn = insns; + *insn++ = BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_6, 63); + *insn++ = BPF_MOV64_REG(BPF_REG_7, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 63); + *insn++ = BPF_MOV64_IMM(BPF_REG_8, 2); + *insn++ = BPF_MOV64_IMM(BPF_REG_9, 0); + insn = fill(insn, front, BPF_REG_8); + *insn++ = BPF_JMP_REG(BPF_JEQ, BPF_REG_6, BPF_REG_9, 1); + *insn++ = BPF_EXIT_INSN(); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_8); + insn = fill(insn, mid, BPF_REG_9); + *insn++ = BPF_JMP_REG(BPF_JNE, BPF_REG_7, BPF_REG_9, 2); + *insn++ = MAY_GOTO(off); + *insn++ = BPF_JMP_A(-2); + insn = fill(insn, back, BPF_REG_8); + *insn++ = BPF_EXIT_INSN(); + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, insn - insns, &opts); + free(insns); + if (!ASSERT_GE(fd, 0, "prog_load")) + return; + + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "retval"); + close(fd); +} + +void test_may_goto_far(void) +{ + if (test__start_subtest("32762")) + test_far(32762); + if (test__start_subtest("32767")) + test_far(32767); + if (test__start_subtest("-32767")) + test_far(-32767); + if (test__start_subtest("-32768")) + test_far(-32768); +} From 6f3ee3516305da3043f1787e3890381a54151730 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:29 +0000 Subject: [PATCH 301/660] bpf: Don't converge a loop on an iterator that was created anew When the verifier reaches bpf_iter_*_next() in a state that is equal to a state that is still being explored it concludes that the loop converged, since an active iterator will be drained eventually. states_equal() matches ids through idmap, so the same is concluded for an iterator that was destroyed and created again since the old state: bpf_iter_num_new(&it, 0, 10); loop: if (bpf_ktime_get_ns()) goto remake; next: if (!bpf_iter_num_next(&it)) goto out; goto loop; remake: bpf_iter_num_destroy(&it); bpf_iter_num_new(&it, 0, 10); goto next; Such prog is accepted when the call of bpf_iter_num_next() is the first prune point after bpf_iter_num_new() and it never returns. Otherwise it's rejected with "infinite loop detected". Require the iterator to have the same id in both states. Fixes: 06accc8779c1 ("bpf: add support for open-coded iterator loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-8-alexei.starovoitov@gmail.com --- kernel/bpf/states.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index c1fbb339329f..18bf7b660c2f 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1335,8 +1335,9 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ if (is_iter_next_insn(env, insn_idx)) { if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { - struct bpf_func_state *cur_frame, *iter_frame; + struct bpf_func_state *cur_frame, *iter_frame, *old_frame; struct bpf_reg_state *iter_state, *iter_reg; + struct bpf_reg_state *old_iter; int spi; cur_frame = cur->frame[cur->curframe]; @@ -1351,7 +1352,16 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) spi = bpf_get_spi(iter_reg->var_off.value); iter_frame = bpf_func(env, iter_reg); iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr; - if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { + old_frame = sl->state.frame[iter_reg->frameno]; + old_iter = &bpf_stack_slot(old_frame, spi)->spilled_ptr; + /* + * states_equal() matches ids through idmap. + * The loop converged only if it's the same + * iterator. The one that was destroyed and + * created again starts from the beginning. + */ + if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE && + iter_state->id == old_iter->id) { loop = true; goto hit; } From 36e238196ac57ba095ba2d9c6f0f715d5c2a054a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 23:31:30 +0000 Subject: [PATCH 302/660] selftests/bpf: Add tests for iterator created anew in its loop Add progs that destroy and create the iterator inside its loop with and without a prune point in front of bpf_iter_num_next(). All of them should be rejected. Nested bpf_for() loops, where the inner iterator is created in every iteration of the outer loop, should load. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260924233130.1213812-9-alexei.starovoitov@gmail.com --- tools/testing/selftests/bpf/progs/iters.c | 382 ++++++++++++++++++++++ 1 file changed, 382 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 65d4c6e01f93..ae845ad747bf 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -2188,4 +2188,386 @@ __naked void loop_counter_precision_2nd_iter(void) ); } +/* + * An iterator that is destroyed and created again inside its loop starts from + * the beginning and the loop never ends. In the first four progs the first + * prune point after bpf_iter_num_new() is the call of bpf_iter_num_next(). + */ +SEC("socket") +__description("iter: remake in the loop, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +__used __naked static void iter_remake(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_iter_num_destroy];" + "r1 = r6;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_destroy) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_callee_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_after_next_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: two iterators, remake each after its next") +__failure __msg("infinite loop detected") +__naked void iter_remake_two_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* Same loops with a prune point in front of bpf_iter_num_next() */ +SEC("socket") +__description("iter: remake in the loop") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_inline(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_callee(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_after_next(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake when drained") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_drained(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 119 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* The inner iterator is created in every iteration of the outer loop */ +SEC("socket") +__description("iter: nested loops") +__success __retval(12) +int iter_nested_remake(void *ctx) +{ + int i, j, sum = 0; + + bpf_for(i, 0, 3) { + bpf_for(j, 0, 4) + sum++; + } + return sum; +} + char _license[] SEC("license") = "GPL"; From 5a84b4424db8ae07f0ac682ab031c37a91dce50b Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 24 Sep 2026 15:13:41 +0200 Subject: [PATCH 303/660] bpf: Fix uninit read for non-fetch atomics on partially spilled slots check_stack_read_fixed_off() skips partial spill checks for non-fetch atomics; the following prog can be loaded: 0: (b7) r1 = 1 ; R1=1 1: (63) *(u32 *)(r10 -8) = r1 ; R1=1 R10=fp0 fp-8=????1 2: (db) lock *(u64 *)(r10 -8) += r1 ; R1=1 R10=fp0 fp-8=mmmmmmmm 3: (79) r0 = *(u64 *)(r10 -8) ; R0=scalar() R10=fp0 fp-8=mmmmmmmm 4: (77) r0 >>= 32 ; R0=scalar(smin=0,smax=umax=0xffffffff,var_off=(0x0; 0xffffffff)) 5: (95) exit When test run: retval=4294967295 Note fp-8 is ????1 at #1, yet it becomes fp-8=mmmmmmmm after the non-fetching atomic add at #2; hence the high 32 bits are leaked. Fix by applying the partial load check; after the patch, the prog is rejected: Verification failed: Memory Safety: Uninitialized stack read Reason: This rejected read uses 8 bytes at stack offset -8, but byte 4 in that range is uninitialized on this path. Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance. At: ... 0 | (b7) r1 = 1 1 | (63) *(u32 *)(r10 -8) = r1 >>> 2 | (db) lock *(u64 *)(r10 -8) += r1 3 | (79) r0 = *(u64 *)(r10 -8) 4 | (77) r0 >>= 32 This affects CAP_BPF only. Fixes: 354e8f1970f8 ("bpf: Support <8-byte scalar spill and refill") Signed-off-by: Hao Sun Link: https://lore.kernel.org/bpf/20260924131342.934290-1-sunhao.th@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 200ec0f71617..c495dc9070de 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4069,11 +4069,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } - if (dst_regno < 0) - return 0; - if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { + if (dst_regno < 0) + return 0; + if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -4115,6 +4115,9 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } + if (dst_regno < 0) + return 0; + if (spill_cnt == size && tnum_is_const(reg->var_off) && reg->var_off.value == 0) { __mark_reg_const_zero(env, &state->regs[dst_regno]); From 51455305b0d6ab5a3432e0d7858f4e73f397cc95 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 24 Sep 2026 15:13:42 +0200 Subject: [PATCH 304/660] selftests/bpf: Test non-fetch atomic on a narrow stack spill Spill a 32-bit scalar into an 8-byte stack slot, leaving the other half of the slot uninitialized, and apply a non-fetch 64-bit atomic add to the slot. With CAP_BPF but without CAP_PERFMON the verifier rejects it. Signed-off-by: Hao Sun Link: https://lore.kernel.org/bpf/20260924131342.934290-2-sunhao.th@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/progs/verifier_spill_fill.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index e1a698db0364..04989d08be44 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1346,6 +1346,23 @@ __naked void old_imprecise_scalar32_vs_cur_stack_misc(void) : __clobber_all); } +SEC("socket") +__description("stack_noperfmon: reject non-fetch atomic on narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+4 size 8") +__naked void stack_noperfmon_reject_atomic_on_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + /* A non-fetch atomic reads all 8 bytes of the slot. */ + "lock *(u64 *)(r10 - 8) += r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + SEC("raw_tp") __success __naked void var_off_write_over_scalar_spill(void) From b3861495d2b2f7459cddf49e63fa0758ba430583 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 25 Sep 2026 10:28:13 +0200 Subject: [PATCH 305/660] bpf: Fix uninit read for helper mem args on partially spilled slots check_stack_range_initialized() takes the spilled register path when bpf_is_spilled_reg(ss) holds, which, however, only looks at the last byte of the slot. Hence, a partially spilled scalar (on the lower part) bypasses the check and the following loads: 0: (b7) r1 = 1 ; R1=1 1: (63) *(u32 *)(r10 -8) = r1 ; R1=1 R10=fp0 fp-8=????1 2: (18) r1 = 0x0 ; R1=map_ptr(map=rb,ks=0,vs=0) 4: (bf) r2 = r10 ; R2=fp0 R10=fp0 5: (07) r2 += -8 ; R2=fp-8 6: (b7) r3 = 8 ; R3=8 7: (b7) r4 = 0 ; R4=0 8: (85) call bpf_ringbuf_output#130 ... When executed, it dumps the uninit four bytes: RINGBUF: 01 00 00 00 ff ff ff ff This is the same root cause as the non-fetch atomic case fixed in commit 5a84b4424db8 ("bpf: Fix uninit read for non-fetch atomics on partially spilled slots"). Fix by only taking the spilled register path for the bytes marked STACK_SPILL, so the rest of the slot goes through the usual checks. Fixes: 354e8f1970f8 ("bpf: Support <8-byte scalar spill and refill") Suggested-by: Kumar Kartikeya Dwivedi Signed-off-by: Hao Sun Link: https://lore.kernel.org/bpf/20260925082814.1554771-1-sunhao.th@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c495dc9070de..5c8626215ee6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7484,7 +7484,12 @@ static int check_stack_range_initialized( goto mark; } - if (bpf_is_spilled_reg(ss) && + /* + * Only the bytes marked STACK_SPILL hold the spilled register. + * The rest of a narrowly spilled slot keeps its previous type + * and must be initialized on its own. + */ + if (*stype == STACK_SPILL && (ss->spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { From 7af653d4ebf8e2e17e288715cd67ccf7845f4631 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 25 Sep 2026 10:28:14 +0200 Subject: [PATCH 306/660] selftests/bpf: Test helper read of a narrow stack spill Spill a 32-bit scalar into an 8-byte stack slot, leaving the other half of the slot uninitialized. With CAP_BPF but without CAP_PERFMON the verifier rejects the helper's read. Signed-off-by: Hao Sun Link: https://lore.kernel.org/bpf/20260925082814.1554771-2-sunhao.th@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/progs/verifier_spill_fill.c | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 04989d08be44..4d1374885f6d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1363,6 +1363,30 @@ __naked void stack_noperfmon_reject_atomic_on_narrow_spill(void) ::: __clobber_all); } +SEC("socket") +__description("stack_noperfmon: reject helper read of narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack R2 off -8+4 size 8") +__naked void stack_noperfmon_reject_helper_read_of_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + "r1 = %[map_ringbuf] ll;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_ringbuf_output];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ringbuf_output), + __imm_addr(map_ringbuf) + : __clobber_all); +} + SEC("raw_tp") __success __naked void var_off_write_over_scalar_spill(void) From b9f30c763c7ecec41fd3f71b8a5a44971c3fddbf Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 08:09:44 +0200 Subject: [PATCH 307/660] bpf: Check callx operand before requiring JIT support check_func_callx() requires a JIT with callx support before it looks at the register operand. On an architecture whose JIT doesn't support callx, or when the JIT is disabled, every callx is thus rejected with "JIT doesn't support callx" or "JIT is required to use callx", including one whose operand is invalid. The "invalid call insn1" test_verifier test expects "R0 !read_ok" for callx of an uninitialized register and fails on s390x: #79/u invalid call insn1 FAIL Unexpected error message! EXP: R0 !read_ok RES: JIT doesn't support callx Validate the operand first and require the JIT only for a callx that is otherwise valid. An invalid callx is then rejected with the same message on every architecture. This fixes various errors when running selftests on JITs where callx is unsupported for now. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925060945.1993547-1-memxor@gmail.com --- kernel/bpf/verifier.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5c8626215ee6..cf9ff37cfdf2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11216,10 +11216,6 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, const char *reason; int err, subprog; - err = require_callx_jit(env); - if (err) - return err; - err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; @@ -11245,6 +11241,11 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; + /* the callx is valid, but only a JIT can execute it */ + err = require_callx_jit(env); + if (err) + return err; + /* PTR_TO_FUNC is a pointer to a static subprog */ subprog = reg->subprogno; err = btf_check_subprog_call(env, subprog, caller->regs); From 3852e19cb9547ffdf9e3fbf3c8decd2c78c85217 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:28 +0200 Subject: [PATCH 308/660] bpf: Avoid quadratic successor rescan in bpf_compute_scc The iterative Tarjan DFS in bpf_compute_scc() emulates recursion with an explicit 'dfs' stack: when a successor has not been visited yet, the successor is pushed and the walk restarts at the top of the loop. On the way back to a vertex the successor scan starts over at index zero, so a vertex with k successors rescans up to k successors on each of its up to k descents, i.e. O(k^2) work. For ordinary instructions k <= 2 and this is irrelevant. For a gotox the successors are the jump table of the containing subprogram, whose size is bounded only by the max_entries of the insn_array map, so k can reach the 1M instruction complexity limit. Loading such a program keeps a CPU busy in the loop for a very long time before verification even begins. Record in 'dfs_pos' the successor index each frame stopped at and resume the scan there. Each edge is therefore examined a bounded number of times and the walk becomes linear in the number of edges. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-2-daniel@iogearbox.net --- kernel/bpf/cfg.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 0de2f634ef67..54ff20234236 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -825,7 +825,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) struct bpf_insn_aux_data *aux = env->insn_aux_data; const u32 insn_cnt = env->prog->len; int stack_sz, dfs_sz, err = 0; - u32 *stack, *pre, *low, *dfs; + u32 *stack, *pre, *low, *dfs, *dfs_pos; u32 i, j, t, w; u32 next_preorder_num; u32 next_scc_id; @@ -838,13 +838,16 @@ int bpf_compute_scc(struct bpf_verifier_env *env) * - 'stack' accumulates vertices in DFS order, see invariant comment below; * - 'pre[t] == p' => preorder number of vertex 't' is 'p'; * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n'; - * - 'dfs' DFS traversal stack, used to emulate explicit recursion. + * - 'dfs' DFS traversal stack, used to emulate explicit recursion; + * - 'dfs_pos[k] == j' => the frame 'dfs[k]' resumes visiting its + * successors at index 'j'. */ stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT); - if (!stack || !pre || !low || !dfs) { + dfs_pos = kvcalloc(insn_cnt, sizeof(*dfs_pos), GFP_KERNEL_ACCOUNT); + if (!stack || !pre || !low || !dfs || !dfs_pos) { err = -ENOMEM; goto exit; } @@ -927,6 +930,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) stack_sz = 0; dfs_sz = 1; dfs[0] = i; + dfs_pos[0] = 0; dfs_continue: while (dfs_sz) { w = dfs[dfs_sz - 1]; @@ -936,13 +940,22 @@ dfs_continue: next_preorder_num++; stack[stack_sz++] = w; } - /* Visit 'w' successors */ + /* Visit remaining 'w' successors */ succ = bpf_insn_successors(env, w); - for (j = 0; j < succ->cnt; ++j) { + for (j = dfs_pos[dfs_sz - 1]; j < succ->cnt; ++j) { if (pre[succ->items[j]]) { low[w] = min(low[w], low[succ->items[j]]); } else { - dfs[dfs_sz++] = succ->items[j]; + /* + * Once DFS for succ->items[j] is complete, + * pre[succ->items[j]] is non-zero, hence + * resuming at 'j' allows to follow the + * low[w] = min(...) update branch above. + */ + dfs_pos[dfs_sz - 1] = j; + dfs_pos[dfs_sz] = 0; + dfs[dfs_sz] = succ->items[j]; + dfs_sz++; goto dfs_continue; } } @@ -992,5 +1005,6 @@ exit: kvfree(pre); kvfree(low); kvfree(dfs); + kvfree(dfs_pos); return err; } From 57f6eca2f63b46dc9526e040cb419321a5013564 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:29 +0200 Subject: [PATCH 309/660] bpf: Bound the number of indirect jump edges in a program Every gotox instruction gets its own copy of the jump table of the subprog containing it, and each distinct target in that table is a CFG successor of the instruction. The number of such edges is therefore the number of gotox instructions times the number of distinct targets, and neither factor is bounded by anything except the instruction limit. What is expensive is a BPF prog whose gotox instructions are themselves the targets, which makes the edge count quadratic. 1024 such gotox are already ~1e6 edges and about 4s of CPU to load. Bound the total across the program at BPF_COMPLEXITY_LIMIT_INSNS, aka the limit on the number of instructions the verifier processes. Progs with real switch statements are orders of magnitude below this. This bounds the edges only. Building the jump table of a gotox costs the number of entries of all insn array maps regardless of how many distinct targets they hold, which a later patch addresses separately. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Link: https://patch.msgid.link/20260925175244.1136329-3-daniel@iogearbox.net --- include/linux/bpf_verifier.h | 1 + kernel/bpf/cfg.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 646e447cebd8..67363c25526d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1004,6 +1004,7 @@ struct bpf_verifier_env { int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; + u32 gotox_edges; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 54ff20234236..69e484b25b59 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -9,6 +9,8 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +#define BPF_MAX_GOTOX_EDGES BPF_COMPLEXITY_LIMIT_INSNS + /* non-recursive DFS pseudo code * 1 procedure DFS-iterative(G,v): * 2 label v as discovered @@ -388,6 +390,19 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) return PTR_ERR(jt); env->insn_aux_data[t].jt = jt; + + if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, + &env->cfg.gotox_edges) || + env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { + verbose(env, "number of indirect jump edges in the program exceeds %u\n", + BPF_MAX_GOTOX_EDGES); + bpf_diag_program_structure( + env, t, "too many indirect jump edges", + "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", + "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", + BPF_MAX_GOTOX_EDGES); + return -E2BIG; + } } mark_prune_point(env, t); From 8608ec8e981974b18834e2d67ed39638a9ec2e63 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:30 +0200 Subject: [PATCH 310/660] bpf: Rebase the exception callback subprog when dead subprogs are removed adjust_subprog_starts_after_remove() compacts subprog_info[] and the func_info when whole subprograms are removed as dead code, but leaves env->exception_callback_subprog alone, so an exception callback declared through the exception_callback decl tag which sits behind a removed subprogram keeps its old index. jit_subprogs() then takes the exception callback from func[] at that index, which is past the end when the callback was the last subprogram, or another subprogram otherwise. Rebase the index along with the entries it refers to. Fixes: b9ae0c9dd0ac ("bpf: Add support for custom exception callbacks") Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-4-daniel@iogearbox.net --- kernel/bpf/fixups.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index e9c2d6c06218..d49c50dae20a 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -506,6 +506,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, * in adjust_btf_func() - no need to adjust */ } + /* The exception callback is live, so it cannot sit inside the removed range. */ + if (env->exception_callback_subprog >= j) + env->exception_callback_subprog -= j - i; } else { /* convert i from "first prog to remove" to "first to adjust" */ if (env->subprog_info[i].start == off) From 939bb6270ee32cfef0631b30ff07a9e0b8806e64 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:31 +0200 Subject: [PATCH 311/660] bpf: Leave out the hidden exit edge of a subprogram without an exit visit_abnormal_return_insn() gives tail_call and ld_{abs,ind} insns a second successor, the exit_idx of their subprogram, so that the hidden exit from the subprogram is part of the CFG. check_subprogs() only assigns exit_idx when it walks over a BPF_EXIT, and a subprogram whose last insn jumps back into itself never has one, so exit_idx stays zero and the edge points at insn 0 of the program: 0: r0 = 0 1: r0 = *(u8 *)skb[0] 2: goto -2 For a subprogram other than main, update_insn() then reads the liveness masks at a negative relative index. Such a subprogram can still load when it leaves through a bpf_throw(), so mark exit_idx as unset in that case and leave the hidden edge out, as there is no exit for it to reach. Fixes: e40f5a6bf88a ("bpf: correct stack liveness for tail calls") Fixes: ee861486e377 ("bpf: Fix ld_{abs,ind} failure path analysis in subprogs") Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-5-daniel@iogearbox.net --- include/linux/bpf_verifier.h | 6 +++++- kernel/bpf/cfg.c | 10 +++++++--- kernel/bpf/verifier.c | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 67363c25526d..0b224498d307 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -815,7 +815,11 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ - u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ + /* + * Index of one of the BPF_EXIT instructions in this subprogram, or + * U32_MAX when it has none. + */ + u32 exit_idx; u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 69e484b25b59..f3f545c53264 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -515,17 +515,21 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) { struct bpf_subprog_info *subprog; struct bpf_iarray *jt; + u32 n; if (env->insn_aux_data[t].jt) return 0; - jt = bpf_iarray_realloc(NULL, 2); + /* A subprogram without an exit has no hidden exit edge to take. */ + subprog = bpf_find_containing_subprog(env, t); + n = subprog->exit_idx == U32_MAX ? 1 : 2; + jt = bpf_iarray_realloc(NULL, n); if (!jt) return -ENOMEM; - subprog = bpf_find_containing_subprog(env, t); jt->items[0] = t + 1; - jt->items[1] = subprog->exit_idx; + if (n == 2) + jt->items[1] = subprog->exit_idx; env->insn_aux_data[t].jt = jt; return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cf9ff37cfdf2..107456fa1845 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2470,6 +2470,7 @@ static int add_subprog(struct bpf_verifier_env *env, int off) return -E2BIG; } /* determine subprog starts. The end is one before the next starts */ + env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; env->subprog_info[env->subprog_cnt++].start = off; sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); From fa5ebade2a54c52b3bf4f9bfd8278238c90ac8fc Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:32 +0200 Subject: [PATCH 312/660] bpf: Cache the jump table of a subprogram during CFG discovery create_jt() builds the jump table of the subprogram containing a gotox by copying out and sorting every insn_array map of the program, and it does so once per gotox instruction. The cost is therefore the number of gotox instructions times the number of entries in all of the maps. A program of 4003 instructions with 2000 gotox and one 500k entry map holding two distinct targets has 4000 indirect jump edges, 0.4% of the limit, and takes 343s to be rejected. The map costs next to nothing to prepare, as an unset entry is already a valid target. At the insn limit, with a single 1M entry map, the same shape extrapolates to 41 hours. All gotox instructions of a subprogram share the same jump table, so build the table of every subprogram in a single pass over the maps and let every gotox use it in place: bpf_insn_successors() resolves a gotox through its containing subprogram, and the tables are kept until the verifier environment is torn down, so nothing is copied into the instruction aux data. The edge accounting in visit_gotox_insn() moves to the first visit of the gotox, marked by the BRANCH bit of its CFG state, and a revisit returns right away since the first one already pushed every target that still needed exploring. The only user left of insn_aux_data[].jt is then the table which visit_abnormal_return_insn() allocates for tail_call and ld_{abs,ind} insns, so that bpf_insn_successors() reports the hidden exit from their subprogram. Both are recognisable by their opcode, so derive that edge in bpf_insn_successors() from the exit_idx of the containing subprogram instead, again leaving it out when the subprogram has no exit, and drop the field along with bpf_clear_insn_aux_data(). The instruction aux data then owns no allocation, thus nothing needs to be freed when insns are removed or the verifier environment is torn down. Subprograms removed as dead code free their table in adjust_subprog_starts_after_remove(), which also clears the slots that the compaction of subprog_info vacates, as they still hold copies of the moved entries and with them their table pointers. check_cfg() is then linear in the number of map entries plus the number of indirect jump edges, so what still scales now with the program is what BPF_MAX_GOTOX_EDGES bounds: gotox map entries edges before after ---------------------------------------------- 500 250000 1000 35.34s 0.07s 1000 250000 2000 82.00s 0.07s 2000 250000 4000 148.62s 0.07s 2000 125000 4000 74.54s 0.05s 2000 500000 4000 342.71s 0.19s Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Suggested-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-6-daniel@iogearbox.net --- include/linux/bpf_verifier.h | 5 +- kernel/bpf/cfg.c | 198 ++++++++++++++++++----------------- kernel/bpf/fixups.c | 30 ++---- kernel/bpf/liveness.c | 26 ++++- kernel/bpf/verifier.c | 3 +- 5 files changed, 138 insertions(+), 124 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 0b224498d307..c775bd757706 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -660,7 +660,6 @@ struct bpf_insn_aux_data { /* remember the offset of node field within type to rewrite */ u64 insert_off; }; - struct bpf_iarray *jt; /* jump table for gotox or bpf_tailcall call instruction */ struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ @@ -820,6 +819,7 @@ struct bpf_subprog_info { * U32_MAX when it has none. */ u32 exit_idx; + struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; @@ -1009,6 +1009,7 @@ struct bpf_verifier_env { /* current position in the insn_postorder vector */ int cur_postorder; u32 gotox_edges; + bool subprog_jts_ready; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; @@ -1587,6 +1588,7 @@ bool bpf_is_throw_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); +void bpf_free_subprog_jts(struct bpf_verifier_env *env); int bpf_compute_postorder(struct bpf_verifier_env *env); int bpf_compute_scc(struct bpf_verifier_env *env); @@ -1805,7 +1807,6 @@ struct bpf_kfunc_desc_tab { }; /* Functions exported from verifier.c, used by fixups.c */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index f3f545c53264..ee89edd09d36 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -286,15 +286,17 @@ err_free: } /* - * Find and collect all maps which fit in the subprog. Return the result as one - * combined jump table in jt->items (allocated with kvcalloc) + * Collect the jump table of every subprogram that has one, as the combined + * table of all maps whose first target lands inside that subprogram. All gotox + * instructions of a subprogram share the same table, so this is done in a + * single pass over the maps rather than once per gotox. */ -static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, - int subprog_start, int subprog_end) +static int __compute_subprog_jts(struct bpf_verifier_env *env) { - struct bpf_iarray *jt = NULL; + struct bpf_subprog_info *subprog; + struct bpf_iarray *jt, *jt_cur; struct bpf_map *map; - struct bpf_iarray *jt_cur; + u32 old_cnt; int i; for (i = 0; i < env->insn_array_map_cnt; i++) { @@ -305,29 +307,86 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, map = env->insn_array_maps[i]; jt_cur = jt_from_map(map); - if (IS_ERR(jt_cur)) { - kvfree(jt); - return jt_cur; - } + if (IS_ERR(jt_cur)) + return PTR_ERR(jt_cur); /* * This is enough to check one element. The full table is - * checked to fit inside the subprog later in create_jt() + * checked to fit inside the subprog later in subprog_jt() */ - if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) { - u32 old_cnt = jt ? jt->cnt : 0; - jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt); - if (!jt) { - kvfree(jt_cur); - return ERR_PTR(-ENOMEM); - } - memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + subprog = bpf_find_containing_subprog(env, jt_cur->items[0]); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map", + jt_cur->items[0])) { + kvfree(jt_cur); + return -EFAULT; } + old_cnt = subprog->jt ? subprog->jt->cnt : 0; + jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); + if (!jt) { + subprog->jt = NULL; + kvfree(jt_cur); + return -ENOMEM; + } + memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + subprog->jt = jt; + kvfree(jt_cur); } - if (!jt) { + for (i = 0; i < env->subprog_cnt; i++) { + jt = env->subprog_info[i].jt; + if (jt) + jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); + } + + env->cfg.subprog_jts_ready = true; + return 0; +} + +/* Leave nothing behind on failure. */ +static int compute_subprog_jts(struct bpf_verifier_env *env) +{ + int err; + + err = __compute_subprog_jts(env); + if (err) + bpf_free_subprog_jts(env); + return err; +} + +void bpf_free_subprog_jts(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(env->subprog_info); i++) { + kvfree(env->subprog_info[i].jt); + env->subprog_info[i].jt = NULL; + } + env->cfg.subprog_jts_ready = false; +} + +static struct bpf_iarray * +subprog_jt(int t, struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog; + int subprog_start, subprog_end; + struct bpf_iarray *jt; + int i, err; + + if (!env->cfg.subprog_jts_ready) { + err = compute_subprog_jts(env); + if (err) + return ERR_PTR(err); + } + + subprog = bpf_find_containing_subprog(env, t); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t)) + return ERR_PTR(-EFAULT); + subprog_start = subprog->start; + subprog_end = (subprog + 1)->start; + + if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); bpf_diag_program_structure( env, subprog_start, "missing jump table", @@ -337,24 +396,7 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, return ERR_PTR(-EINVAL); } - jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); - return jt; -} - -static struct bpf_iarray * -create_jt(int t, struct bpf_verifier_env *env) -{ - struct bpf_subprog_info *subprog; - int subprog_start, subprog_end; - struct bpf_iarray *jt; - int i; - - subprog = bpf_find_containing_subprog(env, t); - subprog_start = subprog->start; - subprog_end = (subprog + 1)->start; - jt = jt_from_subprog(env, subprog_start, subprog_end); - if (IS_ERR(jt)) - return jt; + jt = subprog->jt; /* Check that the every element of the jump table fits within the given subprogram */ for (i = 0; i < jt->cnt; i++) { @@ -366,7 +408,6 @@ create_jt(int t, struct bpf_verifier_env *env) "Keep every jump-table target inside the same subprogram.", "The jump table for instruction %d points outside subprogram range [%u,%u).", t, subprog_start, subprog_end); - kvfree(jt); return ERR_PTR(-EINVAL); } } @@ -383,26 +424,31 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) struct bpf_iarray *jt; int i, w; - jt = env->insn_aux_data[t].jt; - if (!jt) { - jt = create_jt(t, env); - if (IS_ERR(jt)) - return PTR_ERR(jt); + /* + * The first visit pushes every target that still needs exploring, and + * those are all done by the time this insn is revisited, so there is + * nothing left to walk the jump table for. + */ + if (insn_state[t] & BRANCH) + return DONE_EXPLORING; - env->insn_aux_data[t].jt = jt; + jt = subprog_jt(t, env); + if (IS_ERR(jt)) + return PTR_ERR(jt); - if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, - &env->cfg.gotox_edges) || - env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { - verbose(env, "number of indirect jump edges in the program exceeds %u\n", - BPF_MAX_GOTOX_EDGES); - bpf_diag_program_structure( + insn_state[t] |= BRANCH; + + if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, + &env->cfg.gotox_edges) || + env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { + verbose(env, "number of indirect jump edges in the program exceeds %u\n", + BPF_MAX_GOTOX_EDGES); + bpf_diag_program_structure( env, t, "too many indirect jump edges", - "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", - "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", - BPF_MAX_GOTOX_EDGES); - return -E2BIG; - } + "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", + "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", + BPF_MAX_GOTOX_EDGES); + return -E2BIG; } mark_prune_point(env, t); @@ -506,34 +552,6 @@ static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, return DONE_EXPLORING; } -/* - * Instructions that can abnormally return from a subprog (tail_call - * upon success, ld_{abs,ind} upon load failure) have a hidden exit - * that the verifier must account for. - */ -static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) -{ - struct bpf_subprog_info *subprog; - struct bpf_iarray *jt; - u32 n; - - if (env->insn_aux_data[t].jt) - return 0; - - /* A subprogram without an exit has no hidden exit edge to take. */ - subprog = bpf_find_containing_subprog(env, t); - n = subprog->exit_idx == U32_MAX ? 1 : 2; - jt = bpf_iarray_realloc(NULL, n); - if (!jt) - return -ENOMEM; - - jt->items[0] = t + 1; - if (n == 2) - jt->items[1] = subprog->exit_idx; - env->insn_aux_data[t].jt = jt; - return 0; -} - /* Visits the instruction at index t and returns one of the following: * < 0 - an error occurred * DONE_EXPLORING - the instruction was fully explored @@ -556,13 +574,6 @@ static int visit_insn(int t, struct bpf_verifier_env *env) /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { - if (BPF_CLASS(insn->code) == BPF_LD && - (BPF_MODE(insn->code) == BPF_ABS || - BPF_MODE(insn->code) == BPF_IND)) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; return push_insn(t, t + insn_sz, FALLTHROUGH, env); } @@ -607,11 +618,6 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_subprog_might_sleep(env, t); if (bpf_helper_changes_pkt_data(insn->imm)) mark_subprog_changes_pkt_data(env, t); - if (insn->imm == BPF_FUNC_tail_call) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_call_arg_meta meta; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d49c50dae20a..adc254b1a646 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -480,7 +480,11 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, if (j > i) { struct bpf_prog_aux *aux = env->prog->aux; - int move; + int move, k; + + /* The removed subprogs own their jump tables. */ + for (k = i; k < j; k++) + kvfree(env->subprog_info[k].jt); /* move fake 'exit' subprog as well */ move = env->subprog_cnt + 1 - j; @@ -488,6 +492,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, memmove(env->subprog_info + i, env->subprog_info + j, sizeof(*env->subprog_info) * move); + /* The vacated tail still holds copies of the moved entries. */ + memset(env->subprog_info + i + move, 0, + sizeof(*env->subprog_info) * (j - i)); env->subprog_cnt -= j - i; /* remove func_info and its aux */ @@ -586,23 +593,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, return 0; } -/* - * Clean up dynamically allocated fields of aux data for instructions [start, ...] - */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) -{ - struct bpf_insn_aux_data *aux_data = env->insn_aux_data; - int end = start + len; - int i; - - for (i = start; i < end; i++) { - if (aux_data[i].jt) { - kvfree(aux_data[i].jt); - aux_data[i].jt = NULL; - } - } -} - static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; @@ -615,8 +605,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); - bpf_clear_insn_aux_data(env, off, cnt); - err = bpf_remove_insns(env->prog, off, cnt); if (err) return err; @@ -1688,6 +1676,8 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat env->prog = prog; info[cnt + 1].start = info[cnt].start; info[cnt].start = prog->len - len + 1; + /* The patch ends with the exit. */ + info[cnt].exit_idx = prog->len - 1; env->subprog_cnt++; env->hidden_subprog_cnt++; return 0; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 16fb7dc43726..cd9523f69298 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -350,14 +350,18 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) struct bpf_iarray *succ, *jt; int insn_sz; - jt = env->insn_aux_data[idx].jt; - if (unlikely(jt)) - return jt; - /* pre-allocated array of size up to 2; reset cnt, as it may have been used already */ succ = env->succ; succ->cnt = 0; + /* All gotox of a subprogram share its jump table, see compute_subprog_jts() */ + if (unlikely(insn_is_gotox(insn))) { + jt = bpf_find_containing_subprog(env, idx)->jt; + if (verifier_bug_if(!jt, env, "no jump table for gotox insn %u", idx)) + return succ; + return jt; + } + opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)]; insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; if (opcode_info->can_fallthrough) @@ -366,6 +370,20 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) if (opcode_info->can_jump) succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1; + /* + * tail_call upon success and ld_{abs,ind} upon load failure return + * from the subprogram, which is a hidden edge to its exit. A subprogram + * without an exit insn has no exit to model. + */ + if (unlikely((bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) || + (BPF_CLASS(insn->code) == BPF_LD && + (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)))) { + u32 exit_idx = bpf_find_containing_subprog(env, idx)->exit_idx; + + if (exit_idx != U32_MAX) + succ->items[succ->cnt++] = exit_idx; + } + return succ; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 107456fa1845..24f74fd2477f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22757,8 +22757,7 @@ err_prep: release_maps(env); release_btfs(env); err_free_env: - if (env->insn_aux_data) - bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); + bpf_free_subprog_jts(env); vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env); From 19581372dd81f3337bb672a437e56050a3ea41cb Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:33 +0200 Subject: [PATCH 313/660] bpf: Reject indirect jumps that leave their subprogram The jump table of a subprog is collected in compute_subprog_jts() from the insn_array maps of the program, and a map is attributed to the subprog that contains its first entry. check_indirect_jump() instead resolves the targets from the map the gotox register actually points to, bounded only by the index range of that register, and never relates them back to the subprog of the gotox. The two disagree, so bpf_insn_successors() reports a subset of the edges the BPF program can take and a gotox can enter a subprog the CFG never walked. Close both ends: reject a map that reaches past the subprog holding its first entry right when it is collected in compute_subprog_jts(), so that every map of a program with a gotox is confined to one subprog, and confine the resolved targets of a gotox to its subprog in check_indirect_jump(). A target inside the subprog is then always in the jump table the CFG walked, which is the invariant that actually has to hold. The out-of-range check in visit_gotox_insn() cannot fire any more once every table is confined and becomes a verifier_bug_if(). Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Reported-by: James Burton Reported-by: Nuoqi Gui Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-7-daniel@iogearbox.net --- kernel/bpf/cfg.c | 52 ++++++------------- kernel/bpf/verifier.c | 33 ++++++++++++ .../selftests/bpf/progs/verifier_gotox.c | 2 +- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index ee89edd09d36..b0bd9ba951df 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -287,7 +287,7 @@ err_free: /* * Collect the jump table of every subprogram that has one, as the combined - * table of all maps whose first target lands inside that subprogram. All gotox + * table of all maps whose targets land inside that subprogram. All gotox * instructions of a subprogram share the same table, so this is done in a * single pass over the maps rather than once per gotox. */ @@ -310,16 +310,23 @@ static int __compute_subprog_jts(struct bpf_verifier_env *env) if (IS_ERR(jt_cur)) return PTR_ERR(jt_cur); - /* - * This is enough to check one element. The full table is - * checked to fit inside the subprog later in subprog_jt() - */ subprog = bpf_find_containing_subprog(env, jt_cur->items[0]); if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map", jt_cur->items[0])) { kvfree(jt_cur); return -EFAULT; } + if (jt_cur->items[jt_cur->cnt - 1] >= (subprog + 1)->start) { + verbose(env, "jump table of subprog starting at %u spans multiple subprogs, map id=%d\n", + subprog->start, map->id); + bpf_diag_program_structure( + env, subprog->start, "jump table spans subprograms", + "Keep every entry of a jump table inside one subprogram.", + "The jump table in map id %d for the subprogram that starts at instruction %u reaches past its end at instruction %u.", + map->id, subprog->start, (subprog + 1)->start); + kvfree(jt_cur); + return -EINVAL; + } old_cnt = subprog->jt ? subprog->jt->cnt : 0; jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); @@ -370,9 +377,7 @@ static struct bpf_iarray * subprog_jt(int t, struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog; - int subprog_start, subprog_end; - struct bpf_iarray *jt; - int i, err; + int subprog_start, err; if (!env->cfg.subprog_jts_ready) { err = compute_subprog_jts(env); @@ -384,7 +389,6 @@ subprog_jt(int t, struct bpf_verifier_env *env) if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t)) return ERR_PTR(-EFAULT); subprog_start = subprog->start; - subprog_end = (subprog + 1)->start; if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); @@ -396,23 +400,7 @@ subprog_jt(int t, struct bpf_verifier_env *env) return ERR_PTR(-EINVAL); } - jt = subprog->jt; - - /* Check that the every element of the jump table fits within the given subprogram */ - for (i = 0; i < jt->cnt; i++) { - if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { - verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", - t, subprog_start, subprog_end); - bpf_diag_program_structure( - env, t, "jump table target out of range", - "Keep every jump-table target inside the same subprogram.", - "The jump table for instruction %d points outside subprogram range [%u,%u).", - t, subprog_start, subprog_end); - return ERR_PTR(-EINVAL); - } - } - - return jt; + return subprog->jt; } /* "conditional jump with N edges" */ @@ -454,15 +442,9 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) mark_prune_point(env, t); for (i = 0; i < jt->cnt; i++) { w = jt->items[i]; - if (w < 0 || w >= env->prog->len) { - verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); - bpf_diag_program_structure( - env, t, "indirect jump out of range", - "Keep indirect jump targets inside the program.", - "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", - t, w, env->prog->len - 1); - return -EINVAL; - } + if (verifier_bug_if(w < 0 || w >= env->prog->len, env, + "indirect jump from insn %d to %d out of range", t, w)) + return -EFAULT; mark_jmp_point(env, w); mark_jump_target(env, w); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 24f74fd2477f..03dbc0e00398 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19259,10 +19259,25 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env, return 0; } +static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target, + u32 subprog_start, u32 subprog_end) +{ + verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n", + env->insn_idx, target, subprog_start, subprog_end); + bpf_diag_program_structure( + env, env->insn_idx, "indirect jump leaves subprogram", + "Keep every reachable jump-table target inside the subprogram of the indirect jump.", + "Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).", + env->insn_idx, target, subprog_start, subprog_end); + return -EINVAL; +} + /* gotox *dst_reg */ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn) { struct bpf_verifier_state *other_branch; + struct bpf_subprog_info *subprog; + u32 subprog_start, subprog_end; struct bpf_reg_state *dst_reg; struct bpf_map *map; u32 min_index, max_index; @@ -19306,6 +19321,24 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in return -EINVAL; } + subprog = bpf_find_containing_subprog(env, env->insn_idx); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx)) + return -EFAULT; + subprog_start = subprog->start; + subprog_end = (subprog + 1)->start; + + /* + * Maps are confined to a subprog, see compute_subprog_jts(), and the + * targets are sorted and free of duplicates, see sort_insn_array_uniq(), + * so either all of them belong to this subprog or none of them does. + */ + if (env->gotox_tmp_buf->items[0] < subprog_start) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0], + subprog_start, subprog_end); + if (env->gotox_tmp_buf->items[n - 1] >= subprog_end) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1], + subprog_start, subprog_end); + for (i = 0; i < n - 1; i++) { mark_indirect_target(env, env->gotox_tmp_buf->items[i]); other_branch = push_stack(env, env->gotox_tmp_buf->items[i], diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 0e27c2c79c57..4931a77ba0b2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -366,7 +366,7 @@ __used static int test_subprog(void) } SEC("socket") -__failure __msg("jump table for insn 4 points outside of the subprog [0,10]") +__failure __msg("jump table of subprog starting at 0 spans multiple subprogs") __naked void jump_table_outside_subprog(void) { asm volatile (" \ From c48a91c049fd98fcde54880a24c51b9a05f7e185 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:34 +0200 Subject: [PATCH 314/660] bpf: Don't remove indirect jump targets in the nop removal pass bpf_opt_remove_nops() drops every ja +0 and may_goto +0 unconditionally. A gotox target may itself be such a transparent nop: it is a verified entry of an insn array jump table that is reachable at run time through the indirect jump. bpf_insn_array_adjust_after_remove() marks it as INSN_DELETED instead, and bpf_insn_array_ready() skips INSN_DELETED entries, so the program loads with a hole in its jump table. The x86 JIT, for example, lowers gotox to a raw indirect jump, so executing that index jumps to the zeroed ips[] slot, that is, to address 0. Fix by keeping an indirect jump target in place. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Reported-by: Sandipan Roy Reported-by: James Burton Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-8-daniel@iogearbox.net --- kernel/bpf/fixups.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index adc254b1a646..d4e54fa42dae 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -727,6 +727,7 @@ int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) int bpf_opt_remove_nops(struct bpf_verifier_env *env) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; @@ -738,6 +739,8 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) if (!is_may_goto_0 && !is_ja) continue; + if (aux[i].indirect_target) + continue; err = verifier_remove_insns(env, i, 1); if (err) From 9aa11c4656b7bd8a32869296603f117a49eadd6e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:35 +0200 Subject: [PATCH 315/660] bpf: Clear insn array jitted pointers on map reuse An insn array records, per entry, the xlated offset, the jitted offset and a jitted target pointer in ips[]. bpf_insn_array_init() runs when the map is bound to a program and resets only xlated_off; the jitted offset and the ips[] pointer are left as the previous owner set them. A program can be verified and JITed and only then fail to load (e.g. bpf_check_tail_call()). By that point bpf_prog_update_insn_ptrs() has already filled ips[] with pointers into that program's JIT image. Thus, the map is handed back for reuse with the stale pointers intact. If the next program reuses the map, the stale pointer survives in the now active program's jump table, pointing into the previous, freed image. Fix by resetting the jitted offset and the ips[] pointer alongside the xlated offset. Fixes: b4ce5923e780 ("bpf, x86: add new map type: instructions array") Reported-by: Sandipan Roy Reported-by: James Burton Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-9-daniel@iogearbox.net --- kernel/bpf/bpf_insn_array.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index a2f84afe6f7c..5874abd1f743 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -199,12 +199,16 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog) return -EBUSY; /* - * Reset all the map indexes to the original values. This is needed, - * e.g., when a replay of verification with different log level should - * be performed. + * Reset the map to its pre-verification state. The xlated and jitted + * offsets and the jitted target pointers are recomputed by the verifier + * and the JIT for this program, so any values left by a previous owner + * must be cleared here. */ - for (i = 0; i < map->max_entries; i++) + for (i = 0; i < map->max_entries; i++) { values[i].xlated_off = values[i].orig_off; + values[i].jitted_off = 0; + insn_array->ips[i] = 0; + } return 0; } From a83512036e0809497ee3259ba7458de287949b94 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:36 +0200 Subject: [PATCH 316/660] bpf: Clear the insn array used flag with release semantics A program that fails after the JIT ran, for example in bpf_check_tail_call(), has already stored its jitted offsets and target pointers into the insn array maps it uses through bpf_prog_update_insn_ptrs() when release_insn_arrays() hands them back by clearing the used flag with a plain atomic_set(). Nothing orders those stores before the flag, so on a weakly ordered architecture they can still be in flight when the next program takes the map: it is acquired with atomic_xchg() in bpf_insn_array_init() and reset there, and the late stores then land on top of whatever the new owner has written since, be it that reset or the pointers its own JIT stored, leaving entries that point into an image which is about to be freed. Clear the flag with atomic_set_release() instead, so that everything the releasing program stored is visible before the map can be acquired again. This pairs with the fully ordered atomic_xchg() on the acquire side. Fixes: b4ce5923e780 ("bpf, x86: add new map type: instructions array") Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-10-daniel@iogearbox.net --- kernel/bpf/bpf_insn_array.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index 5874abd1f743..61d6a076dfdb 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -232,7 +232,8 @@ void bpf_insn_array_release(struct bpf_map *map) { struct bpf_insn_array *insn_array = cast_insn_array(map); - atomic_set(&insn_array->used, 0); + /* Paired with atomic_xchg() in bpf_insn_array_init(). */ + atomic_set_release(&insn_array->used, 0); } void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) From fba62d33c1254ee706c724e2b3b72fc6e237b7ad Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:37 +0200 Subject: [PATCH 317/660] bpf: Retarget indirect jump targets across prologue prepends A few patches do not expand in place, they prepend: bpf_convert_ctx_accesses() puts the ctx save for gen_epilogue and the instructions of gen_prologue in front of insn 0, and bpf_do_misc_fixups() prepends the may_goto counter init at the start of every subprog that uses may_goto. They copy the original instruction of 'tgt_idx' into the last slot of the patch buffer, so it now lives at tgt_idx + delta, and call adjust_jmp_off() to move direct branches from tgt_idx to tgt_idx + delta. Nothing does the same for indirect branches, so a BPF_MAP_TYPE_INSN_ARRAY slot that named tgt_idx keeps naming tgt_idx, which is now the first prepended instruction, and insn_aux_data[tgt_idx].indirect_target makes the JIT emit the landing pad there. Add a __bpf_patch_insn_data() variant and pass BPF_PATCH_MOVE_TARGET at the affected call-sites to fix up the delta. The poke descriptors of direct tail calls are shifted the same way, and the original instruction is not searched for by content in that mode, as it is the last slot by construction. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Fixes: 07ae6c130b46 ("bpf: Add helper to detect indirect jump targets") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-11-daniel@iogearbox.net --- include/linux/bpf.h | 2 +- kernel/bpf/bpf_insn_array.c | 4 +-- kernel/bpf/fixups.c | 56 ++++++++++++++++++++++++++----------- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 8594c8aff745..2cf04bba16bb 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4289,7 +4289,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index 61d6a076dfdb..92bbb71576ae 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -236,7 +236,7 @@ void bpf_insn_array_release(struct bpf_map *map) atomic_set_release(&insn_array->used, 0); } -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len) { struct bpf_insn_array *insn_array = cast_insn_array(map); int i; @@ -245,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) return; for (i = 0; i < map->max_entries; i++) { - if (insn_array->values[i].xlated_off <= off) + if (insn_array->values[i].xlated_off < first) continue; if (insn_array->values[i].xlated_off == INSN_DELETED) continue; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d4e54fa42dae..37cf130ebb57 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -214,13 +214,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +enum bpf_patch_mode { + BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ + BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ +}; + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, struct bpf_prog *new_prog, u32 off, u32 cnt, - struct bpf_insn *original_insn) + struct bpf_insn *original_insn, + enum bpf_patch_mode mode) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -253,7 +259,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; - if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + /* + * In BPF_PATCH_MOVE_TARGET mode the original insn is the last + * slot by construction. + */ + if (mode == BPF_PATCH_KEEP_TARGET && + !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { data[i].non_stack_access = data[off + cnt - 1].non_stack_access; data[off + cnt - 1].non_stack_access = false; @@ -277,9 +288,10 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new - * instruction. + * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, + * so the flag already sits where needed. */ - if (data[off + cnt - 1].indirect_target) { + if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; } @@ -299,7 +311,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len } } -static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) +static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) { int i; @@ -307,7 +319,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) return; for (i = 0; i < env->insn_array_map_cnt; i++) - bpf_insn_array_adjust(env->insn_array_maps[i], off, len); + bpf_insn_array_adjust(env->insn_array_maps[i], first, len); } static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) @@ -318,7 +330,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); } -static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) +static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) { struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; @@ -326,7 +338,7 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) for (i = 0; i < sz; i++) { desc = &tab[i]; - if (desc->insn_idx <= off) + if (desc->insn_idx < first) continue; desc->insn_idx += len - 1; } @@ -345,11 +357,14 @@ static bool bpf_rewrite_must_abort(void) return false; } -struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len, + enum bpf_patch_mode mode) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + /* The first slot whose targets move. */ + u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; struct bpf_insn original_insn; if (bpf_rewrite_must_abort()) @@ -375,14 +390,20 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len, &original_insn); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); adjust_subprog_starts(env, off, len); - adjust_insn_arrays(env, off, len); + adjust_insn_arrays(env, first, len); bpf_adjust_func_ptrs(env, off, len); - adjust_poke_descs(new_prog, off, len); + adjust_poke_descs(new_prog, first, len); return new_prog; } +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); +} + /* * insn was moved down by delta insns inside its own patch. Operands relative * to the pc that point in front of the old position did not move with it. @@ -883,7 +904,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -906,7 +928,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; @@ -2666,7 +2689,8 @@ next_insn: } cnt++; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; From db8c58c83f5a651051ce83aa681e3d68ad17503a Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:38 +0200 Subject: [PATCH 318/660] selftests/bpf: Add tests for the indirect jump edge limit Add tests for the indirect jump edge limit to the verifier_gotox suite: two programs whose gotox instructions are their own jump table targets, one in a single subprogram and one split across two, exceed the limit and are rejected, and a chain of gotox which sits exactly at the limit still loads and runs. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t verifier_gotox [...] #644/30 verifier_gotox/too-many-gotox-edges:OK #644/31 verifier_gotox/gotox-edges-at-limit:OK #644/32 verifier_gotox/gotox-edges-across-subprogs:OK #644 verifier_gotox:OK Summary: 1/32 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-12-daniel@iogearbox.net --- .../selftests/bpf/progs/verifier_gotox.c | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 4931a77ba0b2..b917c2e14283 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -457,6 +457,100 @@ __naked void spill_fill_ptr_to_insn(void) : __clobber_all); } +/* + * Emit each table entry next to the instruction it targets. Numeric labels + * can be reused by .rept, while %= keeps table symbols unique per asm block. + * BASE is the code section: libbpf relocates section-relative byte offsets + * to instruction offsets in the loaded program. + */ +#define GOTOX_TABLE_BEGIN(COUNT) \ + ".pushsection .jumptables,\"\",@progbits;" \ + "jt_%=:;" \ + ".global jt_%=;" \ + ".size jt_%=, 8 * (" #COUNT ");" \ + ".popsection;" \ + "r1 = jt_%= ll;" + +#define GOTOX_TABLE_ENTRY(LABEL, BASE) \ + ".pushsection .jumptables,\"\",@progbits;" \ + ".quad " LABEL " - " BASE ";" \ + ".popsection;" + +/* N gotox instructions, each targeting all N gotox plus the final block. */ +#define GOTOX_SELF_TARGETS(N, BASE) \ + GOTOX_TABLE_BEGIN(N + 1) \ + "r0 = 0;" \ + ".rept " #N ";" \ + GOTOX_TABLE_ENTRY("1f", BASE) \ + "1:; .8byte %[gotox];" \ + ".endr;" \ + GOTOX_TABLE_ENTRY("2f", BASE) \ + "2:;" + +#define GOTOX_OPERAND \ + __imm_insn(gotox, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + +/* + * Count edges even when their targets are already discovered. These + * programs deliberately use a scalar: the CFG edge limit must reject them + * before symbolic verification reaches the invalid gotox operand. + */ +SEC("socket") +__description("too-many-gotox-edges") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void too_many_gotox_edges(void) +{ + asm volatile ( + /* 1000 * 1001 = 1,001,000 edges. */ + GOTOX_SELF_TARGETS(1000, "socket") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-at-limit") +__success __retval(0) +__naked void gotox_edges_at_limit(void) +{ + asm volatile ( + /* + * 1000 gotox * 1000 targets = 1,000,000 CFG edges. At run + * time, each block loads the next table entry and jumps to + * the following block, so the program terminates. + */ + GOTOX_TABLE_BEGIN(1000) + ".rept 1000;" + GOTOX_TABLE_ENTRY("1f", "socket") + "r0 = *(u64 *)(r1 + 0);" + "r1 += 8;" + ".8byte %[gotox];" + "1:;" + ".endr;" + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +static __noinline __used __naked void gotox_edges_subprog(void) +{ + asm volatile ( + GOTOX_SELF_TARGETS(750, ".text") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-across-subprogs") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void gotox_edges_across_subprogs(void) +{ + asm volatile ( + /* Each subprog has 563,250 edges; together, 1,126,500. */ + GOTOX_SELF_TARGETS(750, "socket") + "call gotox_edges_subprog;" + "exit;" + : : GOTOX_OPERAND : __clobber_all); +} + #endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; From 5add2c64f4740ca4a1944b50f6427eef219dac8c Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:39 +0200 Subject: [PATCH 319/660] selftests/bpf: Add a test for a tail call in a subprogram without an exit Add a test with a subprogram which leaves through a tail call or a bpf_throw() and never reaches an exit insn, so that the hidden edge of the tail call has no exit to point at. The subprogram accesses its stack, so that the liveness pass walks it. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t verifier_cfg [...] #624/21 verifier_cfg/tail call in a subprogram without an exit:OK #624 verifier_cfg:OK Summary: 1/21 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-13-daniel@iogearbox.net --- .../selftests/bpf/progs/verifier_cfg.c | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 3c3bb03e8217..07388a922e5b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -173,4 +173,42 @@ l1_%=: exit; \ " ::: __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_array SEC(".maps"); + +extern void bpf_throw(__u64 cookie) __ksym; + +/* + * The subprogram leaves through the tail call or the exception and never + * reaches an exit insn, so the hidden edge of the tail call has no exit to + * point at. The stack access makes the liveness pass walk the subprogram, + * and the loop compiles to a single jump back onto itself. + */ +__noinline __used +static int tail_call_or_throw_subprog(void *ctx) +{ + volatile __u32 idx = 0; + + bpf_tail_call(ctx, &map_array, idx); + bpf_throw(0); + for (;;) {} +} + +SEC("tc") +__description("tail call in a subprogram without an exit") +__arch_x86_64 +__arch_arm64 +__success +__naked void tail_call_in_subprog_without_exit(void) +{ + asm volatile (" \ + call tail_call_or_throw_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 17c7d885efe8b2b9d547f16a1aee37cbd8ea4bd7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:40 +0200 Subject: [PATCH 320/660] selftests/bpf: Add tests for indirect jumps across subprograms Add various gotox corner case tests for indirect jumps across subprograms, and one where two subprograms only called from dead code sit in front of the one owning a jump table, so that their removal compacts subprog_info[] along with the tables. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t bpf_insn_array,verifier_gotox [...] #27/10 bpf_insn_array/gotox-jt-spans-subprogs:OK #27/11 bpf_insn_array/gotox-jt-spans-with-own-table:OK #27/12 bpf_insn_array/gotox-target-other-subprog:OK #27/13 bpf_insn_array/gotox-jt-per-subprog:OK #27/14 bpf_insn_array/gotox-jt-dead-subprogs:OK #27/15 bpf_insn_array/gotox-target-subprog-from-main:OK #27/16 bpf_insn_array/gotox-index-slice-other-subprog:OK #27/17 bpf_insn_array/gotox-target-other-global-subprog:OK #27/18 bpf_insn_array/gotox-callback-leaves-subprog:OK #27 bpf_insn_array:OK [...] #644 verifier_gotox:OK Summary: 2/50 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-14-daniel@iogearbox.net --- .../selftests/bpf/prog_tests/bpf_insn_array.c | 541 ++++++++++++++++++ 1 file changed, 541 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 0222a9a5d076..40e183269fb6 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) @@ -453,6 +454,519 @@ cleanup: close(map_fd); } +#define GOTOX_LOG_SZ (256 * 1024) + +/* Load and return the fd, which the caller owns. */ +static int gotox_prog_load_funcs_fd(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + + log[0] = 0; + opts.fd_array = fd_array; + opts.fd_array_cnt = fd_array_cnt; + opts.log_buf = log; + opts.log_size = GOTOX_LOG_SZ; + opts.log_level = 1; + if (fi_cnt) { + opts.prog_btf_fd = btf_fd; + opts.func_info = fi; + opts.func_info_cnt = fi_cnt; + opts.func_info_rec_size = sizeof(*fi); + } + + return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); +} + +/* Load for the outcome only: 0 or the negative errno. */ +static int gotox_prog_load_funcs(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + int prog_fd; + + prog_fd = gotox_prog_load_funcs_fd(insns, insn_cnt, fd_array, fd_array_cnt, log, + btf_fd, fi, fi_cnt); + if (prog_fd < 0) + return prog_fd; + close(prog_fd); + return 0; +} + +static int gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log) +{ + return gotox_prog_load_funcs(insns, insn_cnt, fd_array, fd_array_cnt, log, + -1, NULL, 0); +} + +static int gotox_jt_create_offs(const __u32 *offs, __u32 cnt) +{ + struct bpf_insn_array_value val = {}; + int map_fd; + __u32 i; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, cnt); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return map_fd; + + for (i = 0; i < cnt; i++) { + val.orig_off = offs[i]; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, + "bpf_map_update_elem")) + goto err; + } + + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto err; + + return map_fd; +err: + close(map_fd); + return -1; +} + +#define GOTOX_SUB_START 4 +#define GOTOX_MAIN_TGT 2 +#define GOTOX_SUB_TGT 8 +#define GOTOX_TWO_INSN_CNT 10 + +static void gotox_two_subprogs_fill(struct bpf_insn *insns, __u32 jt_idx, __u32 jt_off) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SUB_START - 1 - 1); + insns[GOTOX_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_EXIT_INSN(); + + /* r1 = &jt[0], by index 'jt_idx' into fd_array */ + insns[GOTOX_SUB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[GOTOX_SUB_START + 1] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, jt_off * 8); + insns[7] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[9] = BPF_EXIT_INSN(); +} + +/* + * Load 'insns' with the two jump tables 'jt_a' and 'jt_b', in this order in + * fd_array, and check the outcome. 'fi' describes the subprograms when given. + */ +static void check_gotox_two_maps(struct bpf_insn *insns, __u32 insn_cnt, + const __u32 *jt_a, __u32 a_cnt, + const __u32 *jt_b, __u32 b_cnt, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt, + int expect_err, const char *expect_msg) +{ + int map_fd[2] = { -1, -1 }; + char *log; + int err; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + map_fd[0] = gotox_jt_create_offs(jt_a, a_cnt); + if (map_fd[0] < 0) + goto free_log; + map_fd[1] = gotox_jt_create_offs(jt_b, b_cnt); + if (map_fd[1] < 0) + goto close_maps; + + if (fi) + err = gotox_prog_load_funcs(insns, insn_cnt, map_fd, 2, log, + btf_fd, fi, fi_cnt); + else + err = gotox_prog_load(insns, insn_cnt, map_fd, 2, log); + if (expect_err) + ASSERT_EQ(err, expect_err, "program should have been rejected"); + else + ASSERT_EQ(err, 0, "bpf(BPF_PROG_LOAD)"); + if (expect_msg) + ASSERT_HAS_SUBSTR(log, expect_msg, "verifier log"); + +close_maps: + close(map_fd[0]); + if (map_fd[1] >= 0) + close(map_fd[1]); +free_log: + free(log); +} + +static void check_gotox_target_other_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 0, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); +} + +static void check_gotox_jt_per_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 1, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + 0, NULL); +} + +#define GOTOX_DEAD_MAIN_LEN 7 +#define GOTOX_DEAD_SUB_LEN 6 +#define GOTOX_DEAD_INSN_CNT (GOTOX_DEAD_MAIN_LEN + 3 * GOTOX_DEAD_SUB_LEN) +#define GOTOX_DEAD_LIVE_CNT 10 + +/* r1 = &jt[0], by index 'jt_idx' into fd_array; r1 = *r1; gotox r1; r0 = 0; exit */ +static void gotox_sub_fill(struct bpf_insn *insns, __u32 start, __u32 jt_idx) +{ + insns[start] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[start + 1] = (struct bpf_insn) { .imm = 0 }; + insns[start + 2] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[start + 3] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[start + 4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[start + 5] = BPF_EXIT_INSN(); +} + +/* + * Two subprograms which are only called from dead code sit in front of the + * live one, and all three own a jump table. Dead code removal drops both in + * one go, which compacts subprog_info[] along with the tables. + */ +static void check_gotox_jt_dead_subprogs(void) +{ + const __u32 sub_start[3] = { GOTOX_DEAD_MAIN_LEN, + GOTOX_DEAD_MAIN_LEN + GOTOX_DEAD_SUB_LEN, + GOTOX_DEAD_MAIN_LEN + 2 * GOTOX_DEAD_SUB_LEN }; + struct bpf_insn insns[GOTOX_DEAD_INSN_CNT]; + int map_fd[3] = { -1, -1, -1 }; + struct bpf_prog_info info = {}; + __u32 len = sizeof(info), tgt, i; + int prog_fd; + char *log; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 0); + insns[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2); + insns[2] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[0] - 2 - 1); + insns[3] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[1] - 3 - 1); + insns[4] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[2] - 4 - 1); + insns[5] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[6] = BPF_EXIT_INSN(); + + for (i = 0; i < ARRAY_SIZE(sub_start); i++) { + gotox_sub_fill(insns, sub_start[i], i); + tgt = sub_start[i] + 4; + map_fd[i] = gotox_jt_create_offs(&tgt, 1); + if (map_fd[i] < 0) + goto close_maps; + } + + prog_fd = gotox_prog_load_funcs_fd(insns, ARRAY_SIZE(insns), map_fd, + ARRAY_SIZE(map_fd), log, -1, NULL, 0); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto close_maps; + + /* Both dead subprograms and the calls to them must be gone. */ + if (ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), + "bpf_prog_get_info_by_fd")) + ASSERT_EQ(info.xlated_prog_len / sizeof(struct bpf_insn), + GOTOX_DEAD_LIVE_CNT, "xlated insn count"); + close(prog_fd); +close_maps: + for (i = 0; i < ARRAY_SIZE(map_fd); i++) + if (map_fd[i] >= 0) + close(map_fd[i]); + free(log); +} + +#define GOTOX_FWD_GOTOX 11 +#define GOTOX_FWD_OWN_TGT 12 +#define GOTOX_FWD_SUB_START 14 +#define GOTOX_FWD_INSN_CNT 16 + +static void gotox_from_main_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_FWD_SUB_START - 1 - 1); + insns[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, + offsetof(struct xdp_md, ingress_ifindex)); + insns[3] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 4); + + /* r1 = &jt_leaves[0], by index 1 into fd_array */ + insns[4] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 1, + }; + insns[5] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[7] = BPF_JMP_A(3); + + /* r1 = &jt_own[0], by index 0 into fd_array */ + insns[8] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_FWD_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_FWD_OWN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); + insns[GOTOX_FWD_SUB_START] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[15] = BPF_EXIT_INSN(); +} + +static void check_gotox_target_subprog_from_main(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "indirect jump from insn 11 to 14 leaves the subprog [0,14)"); +} + +/* + * The only map of the subprog holding the gotox reaches past that subprog, so + * the subprog is left without a jump table at all. + */ +static void check_gotox_jt_spans_subprogs(void) +{ + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_span, ARRAY_SIZE(jt_span), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +/* + * The subprog holding the gotox has a well formed jump table of its own and + * also collects a map that reaches past its end. The spanning map is still + * rejected, even though the subprog is not left without a table. + */ +static void check_gotox_jt_spans_with_own_table(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_span, ARRAY_SIZE(jt_span), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +#define GOTOX_SLICE_SUB_START 6 +#define GOTOX_SLICE_GOTOX 14 +#define GOTOX_SLICE_SUB_TGT 15 +#define GOTOX_SLICE_INSN_CNT 17 + +static void gotox_slice_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SLICE_SUB_START - 1 - 1); + insns[2] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[5] = BPF_EXIT_INSN(); + + insns[GOTOX_SLICE_SUB_START] = + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, + offsetof(struct xdp_md, ingress_ifindex)); + insns[7] = BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 1); + insns[8] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1); + insns[9] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[10] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[11] = (struct bpf_insn) { .imm = 0 }; + insns[12] = BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2); + insns[13] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_SLICE_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SLICE_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[16] = BPF_EXIT_INSN(); +} + +static void check_gotox_index_slice_other_subprog(void) +{ + const __u32 jt_main[] = { 2, 3, 4 }; + const __u32 jt_sub[] = { GOTOX_SLICE_SUB_TGT }; + struct bpf_insn insns[GOTOX_SLICE_INSN_CNT]; + + gotox_slice_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 14 to 3 leaves the subprog [6,17)"); +} + +static int gotox_btf_create(const __u32 *starts, const __u8 *linkage, __u32 cnt, + struct bpf_func_info *fi, struct btf **pbtf) +{ + int int_id, proto_id, id; + struct btf *btf; + char name[24]; + __u32 i; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto err; + + proto_id = btf__add_func_proto(btf, int_id); + if (!ASSERT_GT(proto_id, 0, "btf__add_func_proto")) + goto err; + + for (i = 0; i < cnt; i++) { + snprintf(name, sizeof(name), "gotox_f%u", i); + id = btf__add_func(btf, name, linkage[i], proto_id); + if (!ASSERT_GT(id, 0, "btf__add_func")) + goto err; + fi[i].insn_off = starts[i]; + fi[i].type_id = id; + } + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto err; + + *pbtf = btf; + return btf__fd(btf); +err: + btf__free(btf); + return -1; +} + +static void check_gotox_target_other_global_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_SUB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_GLOBAL }; + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_two_subprogs_fill(insns, 0, 0); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); + btf__free(btf); +} + +#define GOTOX_CB_MAIN_TGT 6 +#define GOTOX_CB_START 8 +#define GOTOX_CB_GOTOX 11 +#define GOTOX_CB_TGT 12 +#define GOTOX_CB_INSN_CNT 14 + +static void gotox_callback_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 1); + /* r2 = &callback */ + insns[1] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_2, + .src_reg = BPF_PSEUDO_FUNC, + .imm = GOTOX_CB_START - 1 - 1, + }; + insns[2] = (struct bpf_insn) { .imm = 0 }; + insns[3] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_4, 0); + insns[5] = BPF_EMIT_CALL(BPF_FUNC_loop); + insns[GOTOX_CB_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[7] = BPF_EXIT_INSN(); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[GOTOX_CB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[GOTOX_CB_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_CB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); +} + +static void check_gotox_callback_leaves_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_CB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_STATIC }; + const __u32 jt_main[] = { GOTOX_CB_MAIN_TGT }; + const __u32 jt_cb[] = { GOTOX_CB_TGT }; + struct bpf_insn insns[GOTOX_CB_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_callback_fill(insns); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_cb, ARRAY_SIZE(jt_cb), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 11 to 6 leaves the subprog [8,14)"); + btf__free(btf); +} + static void check_bpf_side(void) { check_bpf_no_lookup(); @@ -490,6 +1004,33 @@ static void __test_bpf_insn_array(void) if (test__start_subtest("bpf-side-ops")) check_bpf_side(); + + if (test__start_subtest("gotox-jt-spans-subprogs")) + check_gotox_jt_spans_subprogs(); + + if (test__start_subtest("gotox-jt-spans-with-own-table")) + check_gotox_jt_spans_with_own_table(); + + if (test__start_subtest("gotox-target-other-subprog")) + check_gotox_target_other_subprog(); + + if (test__start_subtest("gotox-jt-per-subprog")) + check_gotox_jt_per_subprog(); + + if (test__start_subtest("gotox-jt-dead-subprogs")) + check_gotox_jt_dead_subprogs(); + + if (test__start_subtest("gotox-target-subprog-from-main")) + check_gotox_target_subprog_from_main(); + + if (test__start_subtest("gotox-index-slice-other-subprog")) + check_gotox_index_slice_other_subprog(); + + if (test__start_subtest("gotox-target-other-global-subprog")) + check_gotox_target_other_global_subprog(); + + if (test__start_subtest("gotox-callback-leaves-subprog")) + check_gotox_callback_leaves_subprog(); } #else static void __test_bpf_insn_array(void) From 2cc0a520a8e767a787c1cc240bf497a7ea02f8b7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:41 +0200 Subject: [PATCH 321/660] selftests/bpf: Add a test for gotox nop targets Add a test to the verifier_gotox suite which exercises a gotox target which is a ja +0, and checks that the nop removal pass keeps it. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t verifier_gotox [...] #644/33 verifier_gotox/jump_table_target_nop:OK #644 verifier_gotox:OK Summary: 1/33 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-15-daniel@iogearbox.net --- .../selftests/bpf/progs/verifier_gotox.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index b917c2e14283..f5a9878c7b8d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -551,6 +551,35 @@ __naked void gotox_edges_across_subprogs(void) : : GOTOX_OPERAND : __clobber_all); } +/* + * The gotox target is a ja +0, which the nop removal pass must keep as the + * jump table still names it. + */ +SEC("socket") +__success __retval(0) +__xlated("goto pc+0") +__naked void jump_table_target_nop(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad nop_%= - socket; \ + .size jt0_%=, 8; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +nop_%=: \ + goto +0; \ + r0 = 0; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + #endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; From 28e458bbb44b66c991ea20d46141b78df7cb37a6 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:42 +0200 Subject: [PATCH 322/660] selftests/bpf: Add tests for gotox targets across prologue prepends Add tests to the bpf_insn_array suite for a gotox whose jump table points at the first instruction of a subprogram. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t bpf_insn_array [...] #27/19 bpf_insn_array/gotox-target-prologue-shift:OK #27/20 bpf_insn_array/gotox-target-ctx-prologue-shift:OK #27 bpf_insn_array:OK Summary: 1/20 PASSED, 0 SKIPPED, 0/0 FAILED As small nit, cleanup the extra whitespaces between subtests. Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Link: https://patch.msgid.link/20260925175244.1136329-16-daniel@iogearbox.net --- .../selftests/bpf/prog_tests/bpf_insn_array.c | 127 +++++++++++++++--- 1 file changed, 108 insertions(+), 19 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 40e183269fb6..bfedeed24ac5 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -972,65 +972,154 @@ static void check_bpf_side(void) check_bpf_no_lookup(); } +/* Return the xlated index of the first insn equal to 'insn', or -1. */ +static int xlated_insn_index(int prog_fd, const struct bpf_insn *insn) +{ + struct bpf_insn *xlated; + int idx = -1; + __u32 cnt, i; + + if (!ASSERT_OK(get_xlated_program(prog_fd, &xlated, &cnt), "get_xlated_program")) + return -1; + for (i = 0; i < cnt; i++) + if (!memcmp(&xlated[i], insn, sizeof(*insn))) { + idx = i; + break; + } + free(xlated); + return idx; +} + +/* + * Load 'insns' with a jump table naming insn 0, which a prologue prepend + * pushes down, and check that the slot followed it. + */ +static void check_gotox_target_prepend(enum bpf_prog_type prog_type, + struct bpf_insn *insns, __u32 insn_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + struct bpf_insn_array_value val = {}; + int prog_fd = -1, map_fd, idx; + __u32 key = 0; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + val.orig_off = 0; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto cleanup; + + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + prog_fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insn_cnt, &opts); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto cleanup; + + idx = xlated_insn_index(prog_fd, &insns[0]); + if (!ASSERT_GT(idx, 0, "insn 0 was prepended to")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &key, &val), 0, "bpf_map_lookup_elem")) + goto cleanup; + ASSERT_EQ(val.xlated_off, (__u32)idx, "gotox target followed insn 0 past the prepend"); +cleanup: + if (prog_fd >= 0) + close(prog_fd); + close(map_fd); +} + +static void check_gotox_target_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_MOV64_IMM(BPF_REG_0, 0), + /* may_goto +4 -> exit block, bounds the loop */ + BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 4, 0), + /* r1 = &jt[0] (insns 2 and 3) */ + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0), /* r1 = ips[0] */ + /* insn 5: gotox r1 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), /* insn 6: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_XDP, insns, ARRAY_SIZE(insns)); +} + +static void check_gotox_target_ctx_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, -8), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, 0x5a5a, 12), /* second visit exits */ + BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0x5a5a), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* r6 = ctx */ + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, offsetof(struct __sk_buff, data)), + BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, offsetof(struct __sk_buff, data_end)), + BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 1), + BPF_JMP_REG(BPF_JGT, BPF_REG_4, BPF_REG_3, 5), + /* insn 9: direct packet write -> tc unclone prologue prepended at insn 0 */ + BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0), + /* r5 = &jt[0] (insns 10 and 11) */ + BPF_LD_IMM64_RAW(BPF_REG_5, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_5, 0), /* r5 = ips[0] */ + /* insn 13: gotox r5 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_5, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), /* insn 14: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns)); +} + static void __test_bpf_insn_array(void) { /* Test if offsets are adjusted properly */ - if (test__start_subtest("one2one")) check_one_to_one_mapping(); - if (test__start_subtest("simple")) check_simple(); - if (test__start_subtest("deletions")) check_deletions(); - if (test__start_subtest("deletions-with-functions")) check_deletions_with_functions(); - if (test__start_subtest("blindness")) check_blindness(); - /* Check all kinds of operations and related restrictions */ - if (test__start_subtest("incorrect-index")) check_incorrect_index(); - if (test__start_subtest("load-unfrozen-map")) check_load_unfrozen_map(); - if (test__start_subtest("no-map-reuse")) check_no_map_reuse(); - if (test__start_subtest("bpf-side-ops")) check_bpf_side(); - if (test__start_subtest("gotox-jt-spans-subprogs")) check_gotox_jt_spans_subprogs(); - if (test__start_subtest("gotox-jt-spans-with-own-table")) check_gotox_jt_spans_with_own_table(); - if (test__start_subtest("gotox-target-other-subprog")) check_gotox_target_other_subprog(); - if (test__start_subtest("gotox-jt-per-subprog")) check_gotox_jt_per_subprog(); - if (test__start_subtest("gotox-jt-dead-subprogs")) check_gotox_jt_dead_subprogs(); - if (test__start_subtest("gotox-target-subprog-from-main")) check_gotox_target_subprog_from_main(); - if (test__start_subtest("gotox-index-slice-other-subprog")) check_gotox_index_slice_other_subprog(); - if (test__start_subtest("gotox-target-other-global-subprog")) check_gotox_target_other_global_subprog(); - if (test__start_subtest("gotox-callback-leaves-subprog")) check_gotox_callback_leaves_subprog(); + if (test__start_subtest("gotox-target-prologue-shift")) + check_gotox_target_prologue_shift(); + if (test__start_subtest("gotox-target-ctx-prologue-shift")) + check_gotox_target_ctx_prologue_shift(); } #else static void __test_bpf_insn_array(void) From 91ed1942edef5dab681fafed2b15af27ce74a7b4 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:43 +0200 Subject: [PATCH 323/660] selftests/bpf: Add a test for an exception callback behind a dead subprog Add a test where a static subprogram only called from dead code sits in front of the exception callback, so that its removal compacts subprog_info[] underneath the callback's index. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t exceptions [...] #129/119 exceptions/exception_cb_behind_dead_subprog:OK #129 exceptions:OK Summary: 1/119 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925175244.1136329-17-daniel@iogearbox.net --- .../selftests/bpf/prog_tests/exceptions.c | 7 ++++ .../bpf/progs/exceptions_dead_subprog.c | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 639866ce09a9..3705db18c1bb 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -6,6 +6,7 @@ #include "exceptions_ext.skel.h" #include "exceptions_fail.skel.h" #include "exceptions_assert.skel.h" +#include "exceptions_dead_subprog.skel.h" static char log_buf[1024 * 1024]; @@ -410,9 +411,15 @@ static void test_exceptions_assertions(void) RUN_TESTS(exceptions_assert); } +static void test_exceptions_dead_subprog(void) +{ + RUN_TESTS(exceptions_dead_subprog); +} + void test_exceptions(void) { test_exceptions_success(); test_exceptions_failure(); test_exceptions_assertions(); + test_exceptions_dead_subprog(); } diff --git a/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c new file mode 100644 index 000000000000..4f8b82a85234 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +/* + * Read-only and frozen, so the verifier sees a constant zero, while clang + * cannot fold the volatile read and keeps the call below. + */ +const volatile int never; + +/* Only called from dead code, hence removed as a whole before the JIT runs. */ +static __noinline int dead_subprog(struct __sk_buff *ctx) +{ + return ctx->len; +} + +__noinline int exception_cb_behind_dead_subprog_cb(u64 cookie) +{ + return 42; +} + +/* + * libbpf appends the exception callback behind the subprograms reached by + * calls, so removing the dead one compacts subprog_info[] underneath the + * callback's index. + */ +SEC("?tc") +__exception_cb(exception_cb_behind_dead_subprog_cb) +__success __retval(42) +int exception_cb_behind_dead_subprog(struct __sk_buff *ctx) +{ + if (never) + return dead_subprog(ctx); + bpf_throw(0); + return 0; +} + +char _license[] SEC("license") = "GPL"; From f17bedcd29517739a6491c1c51cc6a593f6a0332 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 06:55:28 +0200 Subject: [PATCH 324/660] bpf: Skip zero-length stream writes A stream write whose formatted output is empty still allocates a stream element and links it into the stream log. Capacity accounting only counts payload bytes, so such elements never count against the stream limit, and a program that keeps producing empty output allocates elements without bound until a reader drains them. Return success without allocating anything when there is nothing to write, for both bpf_stream_vprintk() and staged writes. Every element in a stream log then carries at least one byte, which later changes rely on when they derive read readiness from published byte counts. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Suggested-by: Emil Tsalapatis Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925045536.1480933-2-memxor@gmail.com --- kernel/bpf/stream.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 2b80a0599865..9829d1ebce70 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -75,8 +75,13 @@ static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) { - int ret = bpf_stream_consume_capacity(stream, len); + int ret; + /* Nothing to publish; do not allocate an element for it. */ + if (!len) + return 0; + + ret = bpf_stream_consume_capacity(stream, len); if (ret) return ret; @@ -333,8 +338,8 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) va_start(args, fmt); len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, len); + /* Exclude NULL byte during push; skip empty output entirely. */ + ret = len ? __bpf_stream_push_str(&ss->log, buf->buf, len) : 0; if (!ret) ss->len += len; bpf_put_buffers(); From e831ff570908c572d7ee5389a7e03a8637d1d8ee Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 06:55:29 +0200 Subject: [PATCH 325/660] bpf: Add file descriptor interface for program streams The existing BPF_PROG_STREAM_READ_BY_FD command only supports polling a program stream through repeated bpf() calls. It cannot block for new data or integrate with poll-based event loops. Add BPF_PROG_STREAM_OPEN to return a read-only, close-on-exec file descriptor for a selected program stream. Reads block by default and BPF_F_STREAM_NONBLOCK, the only accepted flag, provides non-blocking behavior. poll reports readable data and reports hangup once the program has been freed. Like pipes and sockets, the descriptor is not seekable and lseek fails with ESPIPE. A stream descriptor deliberately does not retain the program. Move each stream into a separately refcounted allocation so program teardown can mark it dead and wake descriptor users while outstanding descriptors drain buffered data safely. Readers sample the dead flag before looking for data, so EOF is reported only when the stream was already dead before it was found empty; data published right before teardown is never skipped. Only programs loaded through BPF_PROG_LOAD get streams. Classic BPF filters, JIT subprograms and shim programs never write to one, and kernel-side writers already resolve a subprogram to its main program, so those programs no longer carry stream state. Readiness needs its own counter. Stream capacity is charged before allocation and before an element is published to the stream log, so using that reservation as the read and poll condition can report readable data while no element exists: a blocking reader retries instead of sleeping and a lone non-blocking reader can see POLLIN followed by EAGAIN. Publish bytes with release ordering after adding elements to the lockless log, use acquire loads before consuming them or reporting readiness, limit each read to its readable snapshot and subtract only bytes actually copied. This keeps the aggregate count correct even when concurrent publishers update it out of publication order. With several readers on one stream, readiness remains advisory, as it is for pipes. The capacity counter is kept solely for enforcing the stream size limit. Wakeups are always deferred through irq_work. Stream writers run in whatever context the program runs in: NMI context for perf_event programs, sections with interrupts disabled inside bpf_spin_lock or rqspinlock critical sections since bpf_stream_vprintk() is KF_SPINLOCK_SAFE, and tracing programs attached anywhere in the kernel, including inside the wait queue and epoll code itself. Waking waiters directly from there can deadlock, and no cheap context check covers every case: on PREEMPT_RT, spinlock_t sections do not disable interrupts, so in_nmi() or irqs_disabled() cannot tell such a program apart from a benign one. Queue an irq_work item instead, as bpf_ringbuf does. Queue it only when a publication turns an empty stream readable. Readers block and pollers wait only after finding the stream empty, and the readable count never drops below zero because each read is bounded by its snapshot, so the first publication after such an observation is the one that makes the count positive, and it is the one that queues the wakeup. Publications into a stream that already holds data raise no interrupt, so a program that prints while nobody drains its stream pays for a single irq_work until the stream is emptied again. This matches bpf_ringbuf, which notifies only once the consumer has caught up. Blocking readers and level-triggered pollers re-check the readable count before waiting, so they cannot miss data, and edge-triggered epoll consumers drain until EAGAIN before waiting again, as epoll(7) requires. Synchronize pending work before releasing the final stream reference so the callback cannot outlive the stream, but only when the work was ever queued: irq_work_sync() waits for an RCU grace period on PREEMPT_RT and on architectures without an irq_work interrupt. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925045536.1480933-3-memxor@gmail.com --- include/linux/bpf.h | 14 ++- include/uapi/linux/bpf.h | 39 ++++++ kernel/bpf/core.c | 8 +- kernel/bpf/stream.c | 209 ++++++++++++++++++++++++++++++--- kernel/bpf/syscall.c | 29 +++++ tools/include/uapi/linux/bpf.h | 39 ++++++ 6 files changed, 315 insertions(+), 23 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2cf04bba16bb..4bae3796c42f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1771,12 +1772,18 @@ enum { }; struct bpf_stream { - atomic_t capacity; + refcount_t refcnt; + atomic_t capacity; /* bytes reserved against the stream limit */ + atomic_t readable; /* published bytes available to readers */ struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ + wait_queue_head_t waitq; + struct irq_work notify_work; + bool notify_used; /* notify_work was queued at least once */ + bool dead; }; struct bpf_stream_stage { @@ -1915,7 +1922,7 @@ struct bpf_prog_aux { struct work_struct work; struct rcu_head rcu; }; - struct bpf_stream stream[2]; + struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; struct bpf_map __rcu *st_ops_assoc; }; @@ -4224,9 +4231,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); -void bpf_prog_stream_init(struct bpf_prog *prog); +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags); void bpf_prog_stream_free(struct bpf_prog *prog); int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 0aaa54359aeb..4687c3310996 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -936,6 +936,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1020,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1524,6 +1552,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1950,6 +1983,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index aa72ad3ec689..d3b8b626ec0f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -142,10 +142,6 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->dst_mutex); mutex_init(&fp->aux->st_ops_assoc_mutex); -#ifdef CONFIG_BPF_SYSCALL - bpf_prog_stream_init(fp); -#endif - return fp; } @@ -289,6 +285,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, void __bpf_prog_free(struct bpf_prog *fp) { if (fp->aux) { +#ifdef CONFIG_BPF_SYSCALL + bpf_prog_stream_free(fp); +#endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); mutex_destroy(&fp->aux->st_ops_assoc_mutex); @@ -3086,7 +3085,6 @@ static void bpf_prog_free_deferred(struct work_struct *work) aux = container_of(work, struct bpf_prog_aux, work); #ifdef CONFIG_BPF_SYSCALL bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); - bpf_prog_stream_free(aux->prog); #endif #ifdef CONFIG_CGROUP_BPF if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 9829d1ebce70..83e803ea54ef 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -2,11 +2,15 @@ /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include +#include #include #include #include +#include #include #include +#include +#include static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) { @@ -73,6 +77,51 @@ static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) atomic_sub(len, &stream->capacity); } +static void bpf_stream_notify(struct irq_work *work) +{ + struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work); + + /* + * Writers run in arbitrary program contexts, including NMI and regions + * that already hold wait queue or epoll locks. Wake waiters from + * irq_work instead, where taking those locks is safe. + */ + wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM); +} + +static void bpf_stream_queue_notify(struct bpf_stream *stream) +{ + /* + * Record that the work has been used so that teardown only pays for + * irq_work_sync(), which may wait for an RCU grace period, when a + * callback could actually be in flight. + */ + if (!READ_ONCE(stream->notify_used)) + WRITE_ONCE(stream->notify_used, true); + irq_work_queue(&stream->notify_work); +} + +static int bpf_stream_readable_bytes(struct bpf_stream *stream) +{ + return atomic_read_acquire(&stream->readable); +} + +static void bpf_stream_publish(struct bpf_stream *stream, int len) +{ + /* + * Pairs with atomic_read_acquire() in bpf_stream_readable_bytes(). + * + * Notify only when the stream turns from empty to readable. Readers + * block and pollers wait only after finding it empty, and a read never + * takes the count below zero, so the publication that makes it positive + * is the one they wait for. Publishing into a stream that already holds + * data would only raise an interrupt for waiters that are being woken + * already, or for nobody at all. + */ + if (atomic_add_return_release(len, &stream->readable) == len) + bpf_stream_queue_notify(stream); +} + static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) { int ret; @@ -88,6 +137,8 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l ret = __bpf_stream_push_str(&stream->log, str, len); if (ret) bpf_stream_release_capacity(stream, len); + else + bpf_stream_publish(stream, len); return ret; } @@ -96,7 +147,7 @@ static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bp { if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) return NULL; - return &aux->stream[stream_id - 1]; + return aux->stream[stream_id - 1]; } static void bpf_stream_free_elem(struct bpf_stream_elem *elem) @@ -164,14 +215,16 @@ static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) { - int rem_len = len, cons_len, ret = 0; + int read_len, rem_len, cons_len, ret = 0; struct bpf_stream_elem *elem = NULL; struct llist_node *node; mutex_lock(&stream->lock); + read_len = min(len, bpf_stream_readable_bytes(stream)); + rem_len = read_len; while (rem_len) { - int pos = len - rem_len; + int pos = read_len - rem_len; int chunk, n; bool cont; @@ -193,7 +246,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) /* Keep any successfully copied bytes; -EFAULT only if none. */ elem->consumed_len -= n; rem_len += n; - ret = (len == rem_len) ? -EFAULT : 0; + ret = (read_len == rem_len) ? -EFAULT : 0; break; } @@ -204,8 +257,9 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) bpf_stream_free_elem(elem); } + atomic_sub(read_len - rem_len, &stream->readable); mutex_unlock(&stream->lock); - return ret ? ret : len - rem_len; + return ret ? ret : read_len - rem_len; } int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) @@ -220,6 +274,111 @@ int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, vo return bpf_stream_read(stream, buf, len); } +static bool bpf_stream_has_data(struct bpf_stream *stream) +{ + return bpf_stream_readable_bytes(stream) > 0; +} + +static void bpf_stream_put(struct bpf_stream *stream) +{ + if (refcount_dec_and_test(&stream->refcnt)) { + struct llist_node *list; + + /* Only a stream that ever queued its work can have a callback in flight. */ + if (READ_ONCE(stream->notify_used)) + irq_work_sync(&stream->notify_work); + list = llist_del_all(&stream->log); + bpf_stream_free_list(list); + bpf_stream_free_list(stream->backlog_head); + mutex_destroy(&stream->lock); + kfree(stream); + } +} + +static int bpf_stream_release(struct inode *inode, struct file *file) +{ + bpf_stream_put(file->private_data); + return 0; +} + +static ssize_t bpf_stream_file_read(struct file *file, char __user *buf, size_t len, + loff_t *ppos) +{ + struct bpf_stream *stream = file->private_data; + bool dead; + int ret; + + if (!len) + return 0; + + for (;;) { + /* + * Sample teardown state before looking for data. Nothing is + * published once the program is gone, so finding the stream + * empty after observing dead means EOF. The opposite order could + * report EOF while data published just before teardown is still + * buffered. + */ + dead = smp_load_acquire(&stream->dead); + ret = bpf_stream_read(stream, buf, len); + if (ret) + return ret; + if (dead) + return 0; + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(stream->waitq, + bpf_stream_has_data(stream) || + READ_ONCE(stream->dead)); + if (ret) + return ret; + } +} + +static __poll_t bpf_stream_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_stream *stream = file->private_data; + __poll_t events = 0; + + /* + * poll_wait() only registers the wait queue callback. Register before + * checking persistent state so a concurrent publication or teardown is + * observed either by the callback or by the checks below. + */ + poll_wait(file, &stream->waitq, pts); + if (bpf_stream_has_data(stream)) + events |= EPOLLIN | EPOLLRDNORM; + if (READ_ONCE(stream->dead)) + events |= EPOLLHUP; + return events; +} + +static const struct file_operations bpf_stream_fops = { + .release = bpf_stream_release, + .read = bpf_stream_file_read, + .poll = bpf_stream_poll, +}; + +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags) +{ + struct bpf_stream *stream; + int fd_flags = O_RDONLY | O_CLOEXEC; + int fd; + + stream = bpf_stream_get(stream_id, prog->aux); + if (!stream) + return -ENOENT; + if (flags & BPF_F_STREAM_NONBLOCK) + fd_flags |= O_NONBLOCK; + + refcount_inc(&stream->refcnt); + fd = anon_inode_getfd("bpf-stream", &bpf_stream_fops, stream, fd_flags); + if (fd < 0) + bpf_stream_put(stream); + return fd; +} + __bpf_kfunc_start_defs(); /* @@ -287,28 +446,47 @@ __bpf_kfunc_end_defs(); /* Added kfunc to common_btf_ids */ -void bpf_prog_stream_init(struct bpf_prog *prog) +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags) { int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - atomic_set(&prog->aux->stream[i].capacity, 0); - init_llist_head(&prog->aux->stream[i].log); - mutex_init(&prog->aux->stream[i].lock); - prog->aux->stream[i].backlog_head = NULL; - prog->aux->stream[i].backlog_tail = NULL; + struct bpf_stream *stream; + + /* On failure, bpf_prog_stream_free() releases the streams allocated so far. */ + stream = kzalloc_obj(*stream, + bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); + if (!stream) + return -ENOMEM; + + refcount_set(&stream->refcnt, 1); + init_llist_head(&stream->log); + mutex_init(&stream->lock); + init_waitqueue_head(&stream->waitq); + init_irq_work(&stream->notify_work, bpf_stream_notify); + prog->aux->stream[i] = stream; } + return 0; } void bpf_prog_stream_free(struct bpf_prog *prog) { - struct llist_node *list; int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - list = llist_del_all(&prog->aux->stream[i].log); - bpf_stream_free_list(list); - bpf_stream_free_list(prog->aux->stream[i].backlog_head); + struct bpf_stream *stream = prog->aux->stream[i]; + + if (!stream) + continue; + /* + * Pairs with smp_load_acquire() in bpf_stream_file_read(): every + * publication precedes the dead flag, so a reader that observes + * it also observes all buffered data. + */ + smp_store_release(&stream->dead, true); + wake_up_interruptible_poll(&stream->waitq, EPOLLHUP); + bpf_stream_put(stream); + prog->aux->stream[i] = NULL; } } @@ -371,6 +549,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, list = tail; } llist_add_batch(head, tail, &stream->log); + bpf_stream_publish(stream, ss->len); return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e1bcfd43f54f..ac52f4ae414c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3080,6 +3080,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; + err = bpf_prog_stream_init(prog, GFP_USER); + if (err) + goto free_prog; + err = -EFAULT; if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), @@ -6316,6 +6320,28 @@ put_prog: return ret; } +#define BPF_PROG_STREAM_OPEN_LAST_FIELD prog_stream_open.flags + +static int prog_stream_open(union bpf_attr *attr) +{ + struct bpf_prog *prog; + u32 flags = attr->prog_stream_open.flags; + int ret; + + if (CHECK_ATTR(BPF_PROG_STREAM_OPEN)) + return -EINVAL; + if (flags & ~BPF_F_STREAM_NONBLOCK) + return -EINVAL; + + prog = bpf_prog_get(attr->prog_stream_open.prog_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + ret = bpf_prog_stream_new_fd(prog, attr->prog_stream_open.stream_id, flags); + bpf_prog_put(prog); + return ret; +} + static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, bpfptr_t uattr_common, unsigned int size_common) { @@ -6488,6 +6514,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, case BPF_PROG_ASSOC_STRUCT_OPS: err = prog_assoc_struct_ops(&attr); break; + case BPF_PROG_STREAM_OPEN: + err = prog_stream_open(&attr); + break; default: err = -EINVAL; break; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 0aaa54359aeb..4687c3310996 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -936,6 +936,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1020,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1524,6 +1552,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1950,6 +1983,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF From 9054903ab51b8e462bd247d6365c11bc4fce90a2 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 06:55:30 +0200 Subject: [PATCH 326/660] libbpf: Add bpf_prog_stream_open() Expose the BPF_PROG_STREAM_OPEN command through a low-level libbpf wrapper. The options structure carries stream open flags, including non-blocking mode, while leaving room for future extensions. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20260925045536.1480933-4-memxor@gmail.com --- tools/lib/bpf/bpf.c | 19 +++++++++++++++++++ tools/lib/bpf/bpf.h | 24 ++++++++++++++++++++++++ tools/lib/bpf/libbpf.map | 1 + 3 files changed, 44 insertions(+) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index a9de7f107cf7..b49822d212ae 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -1466,6 +1466,25 @@ int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len, return libbpf_err_errno(err); } +int bpf_prog_stream_open(int prog_fd, __u32 stream_id, + const struct bpf_prog_stream_open_opts *opts) +{ + const size_t attr_sz = offsetofend(union bpf_attr, prog_stream_open); + union bpf_attr attr; + int fd; + + if (!OPTS_VALID(opts, bpf_prog_stream_open_opts)) + return libbpf_err(-EINVAL); + + memset(&attr, 0, attr_sz); + attr.prog_stream_open.prog_fd = prog_fd; + attr.prog_stream_open.stream_id = stream_id; + attr.prog_stream_open.flags = OPTS_GET(opts, flags, 0); + + fd = sys_bpf_fd(BPF_PROG_STREAM_OPEN, &attr, attr_sz); + return libbpf_err_errno(fd); +} + int bpf_prog_assoc_struct_ops(int prog_fd, int map_fd, struct bpf_prog_assoc_struct_ops_opts *opts) { diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 490e8cb4ba53..826d9cc9ab65 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -759,10 +759,34 @@ struct bpf_prog_stream_read_opts { * * @return The number of bytes read, on success; negative error code, otherwise * (errno is also set to the error code) + * + * For blocking reads and polling, prefer **bpf_prog_stream_open**. */ LIBBPF_API int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len, struct bpf_prog_stream_read_opts *opts); +struct bpf_prog_stream_open_opts { + size_t sz; + __u32 flags; + size_t :0; +}; +#define bpf_prog_stream_open_opts__last_field flags + +/** + * @brief **bpf_prog_stream_open** opens a file descriptor for a BPF stream of + * a given BPF program. + * + * @param prog_fd FD for the BPF program whose BPF stream is to be opened. + * @param stream_id ID of the BPF stream to be opened. + * @param opts optional options, can be NULL. BPF_F_STREAM_NONBLOCK requests a + * non-blocking descriptor. + * + * @return A new stream FD, on success; negative error code, otherwise (errno + * is also set to the error code) + */ +LIBBPF_API int bpf_prog_stream_open(int prog_fd, __u32 stream_id, + const struct bpf_prog_stream_open_opts *opts); + struct bpf_prog_assoc_struct_ops_opts { size_t sz; __u32 flags; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 18d27d20102e..53b7b591325f 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -459,6 +459,7 @@ LIBBPF_1.7.0 { LIBBPF_1.8.0 { global: bpf_map__attach_cgroup_opts; + bpf_prog_stream_open; bpf_program__add_flags; bpf_program__attach_tracing_multi; bpf_program__clear_flags; From 0abecacb4bfddff99064b855ff28613b786fa435 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 06:55:31 +0200 Subject: [PATCH 327/660] bpftool: Add option to wait for program stream output bpftool prog tracelog { stdout | stderr } PROG dumps the output buffered in a program stream and exits, which is all BPF_PROG_STREAM_READ_BY_FD allows. Waiting for further output means rerunning the command. Add a -w/--wait option that opens the stream with bpf_prog_stream_open() in its default blocking mode and keeps printing output as the program produces it. Flush each chunk as it arrives so redirected output is not held in stdio buffers while the next read blocks. Waiting is opt-in: the default dump keeps using BPF_PROG_STREAM_READ_BY_FD and exits once the buffered output is drained, so existing scripts behave the same on old and new kernels. A stream descriptor does not keep its program alive, so bpftool drops the program descriptor once the stream is open. Waiting then ends with EOF when the program is unloaded and otherwise runs until interrupted, as bpftool prog tracelog already does for the trace pipe. Exit from the SIGINT, SIGHUP and SIGTERM handlers like that command does. A flag set by the handler and checked before each read would miss a signal that lands between the check and the blocking read(), leaving bpftool asleep until the next print. Waiting requires BPF_PROG_STREAM_OPEN. The bpf() syscall fails with EINVAL for an unknown command, which bpftool reports as missing kernel support rather than degrading into a dump. Document the option and offer it in the bash completion. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20260925045536.1480933-5-memxor@gmail.com --- .../bpftool/Documentation/bpftool-prog.rst | 12 +++- tools/bpf/bpftool/bash-completion/bpftool | 2 +- tools/bpf/bpftool/main.c | 7 ++- tools/bpf/bpftool/main.h | 1 + tools/bpf/bpftool/prog.c | 62 ++++++++++++++++--- 5 files changed, 73 insertions(+), 11 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 90fe8c61bf42..17aa0d560e32 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -18,7 +18,8 @@ SYNOPSIS *OPTIONS* := { |COMMON_OPTIONS| | { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } | -{ **-L** | **--use-loader** } | [ { **-S** | **--sign** } **-k** **-i** ] } +{ **-L** | **--use-loader** } | [ { **-S** | **--sign** } **-k** **-i** ] | +{ **-w** | **--wait** } } *COMMANDS* := { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** | @@ -186,6 +187,11 @@ bpftool prog tracelog { stdout | stderr } *PROG* error messages to the standard error stream. This facility should be used only for debugging purposes. + By default, bpftool prints the output buffered so far and exits. With + **-w** or **--wait**, it keeps printing new output as the program produces + it, until the program is unloaded or is hit. Waiting requires a + kernel that supports opening a stream as a file descriptor. + bpftool prog run *PROG* data_in *FILE* [data_out *FILE* [data_size_out *L*]] [ctx_in *FILE* [ctx_out *FILE* [ctx_size_out *M*]]] [repeat *N*] Run BPF program *PROG* in the kernel testing infrastructure for BPF, meaning that the program works on the data and context provided by the @@ -267,6 +273,10 @@ OPTIONS Path to the X.509 certificate file in PEM or DER format, required when signing. +-w, --wait + When dumping a program stream with **bpftool prog tracelog**, wait for new + output instead of exiting once the buffered output has been printed. + EXAMPLES ======== **# bpftool prog show** diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 9d9ced270685..a28af31937ad 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -269,7 +269,7 @@ _bpftool() # Deal with options if [[ ${words[cword]} == -* ]]; then local c='--version --json --pretty --bpffs --mapcompat --debug \ - --use-loader --base-btf --sign -i -k' + --use-loader --base-btf --sign -i -k --wait' COMPREPLY=( $( compgen -W "$c" -- "$cur" ) ) return 0 fi diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 83884b21e708..d15fec71b64e 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -31,6 +31,7 @@ bool block_mount; bool verifier_logs; bool relaxed_maps; bool use_loader; +bool wait_output; struct btf *base_btf; struct hashmap *refs_table; bool sign_progs; @@ -462,6 +463,7 @@ int main(int argc, char **argv) { "nomount", no_argument, NULL, 'n' }, { "debug", no_argument, NULL, 'd' }, { "use-loader", no_argument, NULL, 'L' }, + { "wait", no_argument, NULL, 'w' }, { "sign", no_argument, NULL, 'S' }, { "base-btf", required_argument, NULL, 'B' }, { 0 } @@ -480,7 +482,7 @@ int main(int argc, char **argv) bin_name = "bpftool"; opterr = 0; - while ((opt = getopt_long(argc, argv, "VhpjfLmndSi:k:B:l", + while ((opt = getopt_long(argc, argv, "VhpjfLmndSi:k:B:lw", options, NULL)) >= 0) { switch (opt) { case 'V': @@ -530,6 +532,9 @@ int main(int argc, char **argv) case 'L': use_loader = true; break; + case 'w': + wait_output = true; + break; case 'S': sign_progs = true; use_loader = true; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 48eedcb9f6c0..fa2c877a3cbf 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -89,6 +89,7 @@ extern bool block_mount; extern bool verifier_logs; extern bool relaxed_maps; extern bool use_loader; +extern bool wait_output; extern struct btf *base_btf; extern struct hashmap *refs_table; extern bool sign_progs; diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 24e40dfab469..5e27b22444f2 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1119,21 +1119,66 @@ enum prog_tracelog_mode { TRACE_STDERR, }; +static void exit_stream(int signo) +{ + exit(0); +} + +/* Consumes prog_fd. */ static int prog_tracelog_stream(int prog_fd, enum prog_tracelog_mode mode) { + const struct sigaction act = { .sa_handler = exit_stream }; + const int signals[] = { SIGHUP, SIGINT, SIGTERM }; FILE *file = mode == TRACE_STDOUT ? stdout : stderr; int stream_id = mode == TRACE_STDOUT ? 1 : 2; char buf[512]; - int ret; + unsigned int i; + int fd, ret; - ret = 0; - do { - ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL); - if (ret > 0) - fwrite(buf, sizeof(buf[0]), ret, file); - } while (ret > 0); + if (!wait_output) { + do { + ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL); + if (ret > 0) + fwrite(buf, sizeof(buf[0]), ret, file); + } while (ret > 0); + if (ret < 0) + p_err("failed to read stream: %s", strerror(-ret)); + close(prog_fd); + goto out; + } + fd = bpf_prog_stream_open(prog_fd, stream_id, NULL); + /* + * The stream descriptor does not keep the program alive. Drop the + * program reference so that reads return EOF once the program is gone. + */ + close(prog_fd); + if (fd == -EINVAL) { + /* The bpf() syscall rejects unknown commands with EINVAL. */ + p_err("waiting for stream output is not supported by this kernel"); + return -1; + } + if (fd < 0) { + p_err("failed to open stream: %s", strerror(-fd)); + return -1; + } + + /* + * Exit from the handler like the trace pipe variant does. A flag checked + * between reads would miss a signal that lands before read() blocks and + * only end the loop at the next print. + */ + for (i = 0; i < ARRAY_SIZE(signals); i++) + sigaction(signals[i], &act, NULL); + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + fwrite(buf, sizeof(buf[0]), ret, file); + fflush(file); + } + if (ret < 0) + p_err("failed to read stream: %s", strerror(errno)); + close(fd); +out: fflush(file); return ret ? -1 : 0; } @@ -2659,7 +2704,8 @@ static int do_help(int argc, char **argv) " METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n" - " {-L|--use-loader} | [ {-S|--sign } {-k} {-i} ] \n" + " {-L|--use-loader} | [ {-S|--sign } {-k} {-i} ] |\n" + " {-w|--wait} }\n" "", bin_name, argv[-2]); From 4e1d734651e5a0379defd42c53bd2f985e3e33a6 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 06:55:32 +0200 Subject: [PATCH 328/660] selftests/bpf: Test program stream file descriptors Exercise validation and normal file semantics of BPF_PROG_STREAM_OPEN. Cover read-only and close-on-exec state, write and lseek rejection, non-blocking reads, partial reads, poll readiness, and a blocking reader woken by new stream data. Also check that an empty stream write produces neither readable data nor poll readiness. Register the descriptor with EPOLLET, consume the initial data event without draining buffered output, then release the program. Require a second edge carrying POLLIN and POLLHUP, drain the buffered bytes, and verify that poll continues to report HUP before read returns EOF. This ensures program teardown notifies edge-triggered loops even when readability never transitions. Notifications are only sent when an empty stream turns readable, so also consume the first edge of an edge-triggered descriptor, leave part of the output behind while more arrives, drain the stream and require a new edge for output published afterwards. This covers the transition that re-arms notifications once readers, including partial ones, bring the count back to zero. Also block a reader on an empty live stream, release the program, and verify that it wakes and returns EOF. This directly covers the teardown wait condition and its no-lost-wakeup ordering. Finally, attach a stream-writing program to a hardware perf event and busy-poll its epoll registration until output arrives. This both drives the per-thread event into NMI context and directly verifies that the deferred irq_work notification reaches epoll, without a separate BSS completion handshake. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925045536.1480933-6-memxor@gmail.com --- .../testing/selftests/bpf/prog_tests/stream.c | 486 ++++++++++++++++++ tools/testing/selftests/bpf/progs/stream.c | 20 + 2 files changed, 506 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index 74bd15c4bfba..ae400dcfbf24 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -1,11 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include +#include +#include +#include #include +#include #include "stream.skel.h" #include "stream_fail.skel.h" +#define NMI_TIMEOUT_NS (5ULL * 1000 * 1000 * 1000) + void test_stream_failure(void) { RUN_TESTS(stream_fail); @@ -62,6 +68,486 @@ void test_stream_syscall(void) stream__destroy(skel); } +static bool stream_fd_trigger(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; + + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(ret, "test_run") && ASSERT_OK(opts.retval, "retval"); +} + +static void test_stream_fd_open(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts); + struct stream *skel; + int fd, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + fd = bpf_prog_stream_open(0, BPF_STREAM_STDOUT, NULL); + ASSERT_EQ(fd, -EINVAL, "bad_prog_fd"); + + fd = bpf_prog_stream_open(prog_fd, 0, NULL); + ASSERT_EQ(fd, -ENOENT, "bad_stream_id"); + + opts.flags = BPF_F_RDONLY; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "access_flag"); + + opts.flags = 1U << 31; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "unknown_flag"); + + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDERR, NULL); + if (ASSERT_OK_FD(fd, "stderr")) + close(fd); + + stream__destroy(skel); +} + +static void test_stream_fd_nonblock(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + flags = fcntl(fd, F_GETFD); + ASSERT_GE(flags, 0, "getfd"); + ASSERT_NEQ(flags & FD_CLOEXEC, 0, "cloexec"); + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_ACCMODE, O_RDONLY, "readonly"); + ASSERT_NEQ(flags & O_NONBLOCK, 0, "nonblock"); + + ret = write(fd, "x", 1); + ASSERT_EQ(ret, -1, "write"); + ASSERT_EQ(errno, EBADF, "write_errno"); + ret = lseek(fd, 0, SEEK_SET); + ASSERT_EQ(ret, -1, "lseek"); + ASSERT_EQ(errno, ESPIPE, "lseek_errno"); + + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty"); + ASSERT_EQ(errno, EAGAIN, "read_empty_errno"); + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_data"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin"); + ASSERT_EQ(pfd.revents & POLLHUP, 0, "no_pollhup"); + + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_first"); + ASSERT_OK(memcmp(buf, "fo", 2), "read_first_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_partial"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin_partial"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 1, "read_rest"); + ASSERT_EQ(buf[0], 'o', "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_empty(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4]; + int fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_empty), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + /* Empty output produces neither data nor readiness. */ + if (!stream_fd_trigger(skel->progs.stream_empty)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty_write"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty_write"); + ASSERT_EQ(errno, EAGAIN, "read_empty_write_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +struct stream_fd_read_ctx { + int fd; + ssize_t ret; + char buf[4]; +}; + +static void *stream_fd_read_thread(void *arg) +{ + struct stream_fd_read_ctx *ctx = arg; + + ctx->ret = read(ctx->fd, ctx->buf, sizeof(ctx->buf)); + return NULL; +} + +static int stream_fd_timed_join(pthread_t thread) +{ + struct timespec timeout; + + clock_gettime(CLOCK_REALTIME, &timeout); + timeout.tv_sec += 5; + return pthread_timedjoin_np(thread, NULL, &timeout); +} + +static void test_stream_fd_blocking(void) +{ + struct stream_fd_read_ctx ctx = {}; + struct stream *skel; + pthread_t thread; + bool thread_live = false; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, NULL); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + ctx.fd = fd; + + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_NONBLOCK, 0, "blocking"); + + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_thread; + + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 3, "read_len"); + ASSERT_OK(memcmp(ctx.buf, "foo", 3), "read_data"); + + memset(&ctx, 0, sizeof(ctx)); + ctx.fd = fd; + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create_eof")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_eof_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join_eof")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 0, "read_eof"); + +out_thread: + if (thread_live) { + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (ret) { + pthread_cancel(thread); + pthread_join(thread, NULL); + } + } +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_hup(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_data")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + ASSERT_EQ(event.events & EPOLLHUP, 0, "no_epollhup"); + + stream__destroy(skel); + skel = NULL; + event.events = 0; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_hup")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_with_hup"); + ASSERT_NEQ(event.events & EPOLLHUP, 0, "epollhup"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_buffered"); + ASSERT_OK(memcmp(buf, "foo", 3), "read_buffered_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_drained_hup"); + ASSERT_EQ(pfd.revents & POLLIN, 0, "no_pollin_after_drain"); + ASSERT_NEQ(pfd.revents & POLLHUP, 0, "pollhup_after_drain"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 0, "read_eof"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_edge(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct stream *skel; + char buf[8] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + /* Output into an empty stream raises an edge. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_first")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_first"); + + /* + * Leave data behind, add more, then drain everything. Whether output + * into a non-empty stream raises an edge of its own is not part of the + * contract, so discard any pending event before re-arming the stream. + */ + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_partial"); + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 4, "read_rest"); + ASSERT_OK(memcmp(buf, "ofoo", 4), "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + epoll_wait(epfd, &event, 1, 0); + + /* Output into the drained stream raises an edge again. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_after_drain")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_after_drain"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_nmi_epoll(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .freq = 1, + .sample_freq = 10, + }; + struct epoll_event event = { + .events = EPOLLIN, + }; + struct bpf_link *link = NULL; + struct stream *skel; + __u64 deadline; + char buf[4] = {}; + int epfd = -1, fd = -1, pmu_fd = -1; + int ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_nmi), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out; + + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out; + + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, + PERF_FLAG_FD_CLOEXEC); + if (pmu_fd < 0 && (errno == ENOENT || errno == EOPNOTSUPP)) { + printf("%s:SKIP:no PERF_COUNT_HW_CPU_CYCLES\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) + goto out; + + link = bpf_program__attach_perf_event(skel->progs.stream_nmi, pmu_fd); + if (!ASSERT_OK_PTR(link, "attach_perf_event")) { + link = NULL; + goto out; + } + pmu_fd = -1; + + deadline = get_time_ns() + NMI_TIMEOUT_NS; + do { + ret = epoll_wait(epfd, &event, 1, 0); + } while (ret == 0 && get_time_ns() < deadline); + if (!ASSERT_EQ(ret, 1, "epoll_wait")) + goto out; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_len"); + ASSERT_OK(memcmp(buf, "nmi", 3), "read_data"); + +out: + bpf_link__destroy(link); + if (pmu_fd >= 0) + close(pmu_fd); + if (epfd >= 0) + close(epfd); + if (fd >= 0) + close(fd); + stream__destroy(skel); +} + +void test_stream_fd(void) +{ + if (test__start_subtest("open")) + test_stream_fd_open(); + if (test__start_subtest("nonblock")) + test_stream_fd_nonblock(); + if (test__start_subtest("empty")) + test_stream_fd_empty(); + if (test__start_subtest("blocking")) + test_stream_fd_blocking(); + if (test__start_subtest("hup")) + test_stream_fd_hup(); + if (test__start_subtest("edge")) + test_stream_fd_edge(); + if (test__start_subtest("nmi_epoll")) + test_stream_fd_nmi_epoll(); +} + void test_stream_oversize(void) { LIBBPF_OPTS(bpf_test_run_opts, opts); diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 12fc29e45487..1ae015c4e6b4 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -44,6 +44,7 @@ struct { _X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64) int size; +int nmi_stream_prints; u64 fault_addr; void *arena_ptr; @@ -124,6 +125,25 @@ int stream_syscall(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int stream_empty(void *ctx) +{ + return bpf_stream_printk(BPF_STDOUT, ""); +} + +SEC("perf_event") +int stream_nmi(void *ctx) +{ + if (nmi_stream_prints) + return 0; + /* Retry on a later sample if the write failed, e.g. with -EBUSY. */ + if (bpf_stream_printk(BPF_STDOUT, "nmi")) + return 0; + nmi_stream_prints = 1; + return 0; +} + SEC("syscall") __success __retval(0) int stream_oversize(void *ctx) From 89d8e19811f2b18c273fe4056d193e82afaa6e1a Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Sat, 26 Sep 2026 01:19:31 +0200 Subject: [PATCH 329/660] bpf, mips: Add JIT support for unconditional BSWAP The unconditional BSWAP instruction from the eBPF v4 ISA was not implemented. This caused programs using the instruction to fall back to the interpreter. Implement the unconditional BSWAP instruction. Signed-off-by: Johan Almbladh Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925231932.1063678-2-johan.almbladh@anyfinetworks.com --- arch/mips/net/bpf_jit_comp32.c | 4 ++++ arch/mips/net/bpf_jit_comp64.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index bbbe4a737825..158e69718c09 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1701,6 +1701,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MOD | BPF_X: emit_divmod_r64(ctx, dst, src, BPF_OP(code), off); break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); + break; /* dst = htole(dst) */ /* dst = htobe(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index b04b68fa5ace..8359ad3f6992 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -876,6 +876,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MOD | BPF_X: emit_alu_r64(ctx, dst, src, BPF_OP(code), off); break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); + break; /* dst = htole(dst) */ /* dst = htobe(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: From f4f496decae188ab1621023fecb3d2c191e6e43a Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Sat, 26 Sep 2026 01:19:32 +0200 Subject: [PATCH 330/660] bpf, mips: Add JIT support for JMP32_JA The JMP32_JA instruction from the eBPF v4 ISA was not implemented. This caused programs using the instruction to fall back to the interpreter. Implement the JMP32_JA instruction. Signed-off-by: Johan Almbladh Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925231932.1063678-3-johan.almbladh@anyfinetworks.com --- arch/mips/net/bpf_jit_comp.c | 2 +- arch/mips/net/bpf_jit_comp.h | 2 +- arch/mips/net/bpf_jit_comp32.c | 7 +++++++ arch/mips/net/bpf_jit_comp64.c | 7 +++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index f14de6d73122..4682dbb3a059 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -850,7 +850,7 @@ void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) } /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off) +int emit_ja(struct jit_context *ctx, s32 off) { int target = get_target(ctx, ctx->bpf_index + off + 1); diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h index 0ddedb733dd1..d0ab5c9a0c46 100644 --- a/arch/mips/net/bpf_jit_comp.h +++ b/arch/mips/net/bpf_jit_comp.h @@ -204,7 +204,7 @@ void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off); +int emit_ja(struct jit_context *ctx, s32 off); /* Jump to epilogue */ int emit_exit(struct jit_context *ctx); diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 158e69718c09..84d9c7036e4f 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1877,6 +1877,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */ diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 8359ad3f6992..994589c3de46 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -1041,6 +1041,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */ From d07d3efd0141de96f1f91b379bf9ba6bfcd6dd32 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 25 Sep 2026 17:02:42 -0700 Subject: [PATCH 331/660] libbpf: Fix static linking of externs placed in allocated sections Clang puts an extern variable that ends up with a section, an extern map in .maps or an arena global in .addr_space., into the BTF DATASEC of that section, next to the object's own definitions and with offset 0. Unlike .kconfig and .ksyms, that section is a real allocated ELF section whenever the same object also defines a variable in it. The static linker handles such externs incorrectly in two ways. The output symbol of an unresolved extern gets the destination section's index and offset instead of staying SHN_UNDEF with a zero value. For ephemeral sections both are 0, so this only shows up for allocated sections, where the extern turns into a size-zero NOTYPE definition. Linking the output again then fails with "conflicting non-weak symbol" against a strong definition of that variable, or, against a weak one, keeps the bogus symbol and resolves every reference to offset 0 of the section. And if the section of the object with the extern doesn't start at offset 0 of the output section, the output symbol fails linker_sanity_check_elf_symtab() on the next link with "invalid extern symbol" before any of that. When the extern is resolved by a later object in the same link, the DATASEC entry appended for the extern keeps its placeholder offset, as linker_append_btf() only updates the size of an existing entry. The ELF symbol and the BTF secinfo then disagree and skeleton generation fails with: Error: Something is wrong for .addr_space.1's variable #1: need offset 0, already at 8. Keep unresolved externs SHN_UNDEF with a zero value in the output symbol table, and once an extern is resolved take the offset of its DATASEC entry from the ELF symbol, which already accounts for weak vs. strong resolution. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926000243.2830819-1-andrii@kernel.org --- tools/lib/bpf/linker.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 53f64a1a1f25..f3f71c452f00 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -2159,8 +2159,13 @@ add_sym: dst_sym->st_name = name_off; dst_sym->st_info = sym->st_info; dst_sym->st_other = sym->st_other; - dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; - dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; + if (sym_is_extern) { + dst_sym->st_shndx = SHN_UNDEF; + dst_sym->st_value = 0; + } else { + dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; + dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; + } dst_sym->st_size = sym->st_size; obj->sym_map[src_sym_idx] = dst_sym_idx; @@ -2592,6 +2597,10 @@ static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) continue; dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; + if (!glob_sym->is_extern) { + Elf64_Sym *sym = get_sym_by_idx(linker, glob_sym->sym_idx); + dst_var->offset = sym->st_value; + } /* Because underlying BTF type might have * changed, so might its size have changed, so * re-calculate and update it in sec_var. From 34354db1b148952956518379cded5c25b91e929e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 25 Sep 2026 17:02:43 -0700 Subject: [PATCH 332/660] selftests/bpf: Add linked_externs test for externs in allocated sections Add two objects that define an arena global each and declare the other one's as extern, linked into a single skeleton with the extern-declaring object first. The skeleton build relinks the linked object twice more, which fixes up DATASEC offsets from the ELF symbol table, so the test also drives the libbpf linker directly: it links linked_arena1.bpf.o on its own, links the result with linked_arena2.bpf.o, checks that no variable in the .addr_space.1 DATASEC is left extern, and loads the object. The same is done with linked_maps1/2.bpf.o, which have an extern map in .maps. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926000243.2830819-2-andrii@kernel.org --- tools/testing/selftests/bpf/Makefile.skel | 3 +- .../selftests/bpf/prog_tests/linked_externs.c | 109 ++++++++++++++++++ .../selftests/bpf/progs/linked_arena1.c | 22 ++++ .../selftests/bpf/progs/linked_arena2.c | 18 +++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/linked_externs.c create mode 100644 tools/testing/selftests/bpf/progs/linked_arena1.c create mode 100644 tools/testing/selftests/bpf/progs/linked_arena2.c diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 3d92cdca62ed..2e22bb901bf3 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -23,7 +23,7 @@ BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ - linked_vars.skel.h linked_maps.skel.h \ + linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ test_subskeleton.skel.h test_subskeleton_lib.skel.h \ test_usdt.skel.h tracing_multi.skel.h \ tracing_multi_module.skel.h \ @@ -46,6 +46,7 @@ test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2. linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o +linked_arena.skel.h-deps := linked_arena1.bpf.o linked_arena2.bpf.o # In the subskeleton case, we want the test_subskeleton_lib.subskel.h file # but that's created as a side-effect of the skel.h generation. test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o diff --git a/tools/testing/selftests/bpf/prog_tests/linked_externs.c b/tools/testing/selftests/bpf/prog_tests/linked_externs.c new file mode 100644 index 000000000000..29358c82dc35 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/linked_externs.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "linked_arena.skel.h" + +static void test_skel(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct linked_arena *skel; + int err; + + skel = linked_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + ASSERT_EQ(skel->arena->a_val, 1, "a_val_init"); + ASSERT_EQ(skel->arena->b_val, 2, "b_val_init"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.sum1), &opts); + ASSERT_OK(err, "sum1_run"); + ASSERT_EQ(opts.retval, 1 + 2, "sum1_retval"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.bump2), &opts); + ASSERT_OK(err, "bump2_run"); + ASSERT_EQ(opts.retval, 11 + 22, "bump2_retval"); + + ASSERT_EQ(skel->arena->a_val, 11, "a_val"); + ASSERT_EQ(skel->arena->b_val, 22, "b_val"); + + linked_arena__destroy(skel); +} + +static int link_objs(const char *out, const char *in1, const char *in2) +{ + struct bpf_linker *linker; + int err; + + linker = bpf_linker__new(out, NULL); + if (!ASSERT_OK_PTR(linker, "linker_new")) + return -EINVAL; + + err = bpf_linker__add_file(linker, in1, NULL); + if (!ASSERT_OK(err, in1)) + goto cleanup; + if (in2) { + err = bpf_linker__add_file(linker, in2, NULL); + if (!ASSERT_OK(err, in2)) + goto cleanup; + } + err = bpf_linker__finalize(linker); + ASSERT_OK(err, "finalize"); + +cleanup: + bpf_linker__free(linker); + return err; +} + +/* link in1 on its own first, then link the result with in2 */ +static void test_relink(const char *in1, const char *in2, const char *sec_name) +{ + char out1[] = "/tmp/linked_externs.XXXXXX", out2[] = "/tmp/linked_externs.XXXXXX"; + const struct btf_var_secinfo *vi; + struct bpf_object *obj = NULL; + const struct btf_type *t; + struct btf *btf; + int i, id, n; + + close(mkstemp(out1)); + close(mkstemp(out2)); + + if (!ASSERT_OK(link_objs(out1, in1, NULL), "link_stage1") || + !ASSERT_OK(link_objs(out2, out1, in2), "link_stage2")) + goto cleanup; + + obj = bpf_object__open_file(out2, NULL); + if (!ASSERT_OK_PTR(obj, "obj_open")) + goto cleanup; + + btf = bpf_object__btf(obj); + id = btf__find_by_name_kind(btf, sec_name, BTF_KIND_DATASEC); + if (!ASSERT_GT(id, 0, "find_datasec")) + goto cleanup; + + t = btf__type_by_id(btf, id); + vi = btf_var_secinfos(t); + n = btf_vlen(t); + for (i = 0; i < n; i++) { + t = btf__type_by_id(btf, vi[i].type); + ASSERT_NEQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_EXTERN, "var_resolved"); + } + + ASSERT_OK(bpf_object__load(obj), "obj_load"); + +cleanup: + bpf_object__close(obj); + unlink(out1); + unlink(out2); +} + +void test_linked_externs(void) +{ + if (test__start_subtest("skel_arena")) + test_skel(); + if (test__start_subtest("relink_arena")) + test_relink("linked_arena1.bpf.o", "linked_arena2.bpf.o", ".addr_space.1"); + if (test__start_subtest("relink_maps")) + test_relink("linked_maps1.bpf.o", "linked_maps2.bpf.o", ".maps"); +} diff --git a/tools/testing/selftests/bpf/progs/linked_arena1.c b/tools/testing/selftests/bpf/progs/linked_arena1.c new file mode 100644 index 000000000000..3e8788dec1d9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena1.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); /* number of pages */ +} arena SEC(".maps"); + +long __arena a_val = 1; +extern long __arena b_val; /* defined in linked_arena2.c */ + +SEC("syscall") +int sum1(void *ctx) +{ + return a_val + b_val; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_arena2.c b/tools/testing/selftests/bpf/progs/linked_arena2.c new file mode 100644 index 000000000000..d0bcdf18a4b6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena2.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_arena_common.h" + +long __arena b_val = 2; +extern long __arena a_val; /* defined in linked_arena1.c */ + +SEC("syscall") +int bump2(void *ctx) +{ + a_val += 10; + b_val += 20; + return a_val + b_val; +} + +char _license[] SEC("license") = "GPL"; From 12aa64bbce1247d24f410843e2b71dbc8316e3e7 Mon Sep 17 00:00:00 2001 From: Eva Kurchatova Date: Fri, 25 Sep 2026 18:49:19 +0300 Subject: [PATCH 333/660] samples/bpf: Build the BPF programs with -fms-extensions Commit 835a50753579 ("selftests/bpf: Add -fms-extensions to bpf build flags") gave the selftests the flag because the kernel is built with it, and the generated vmlinux.h carries anonymous members declared by tag: struct slab { ... struct freelist_counters; }; samples/bpf did not get it. clang without the flag reads such a line as a declaration that declares nothing, one warning per line for every sample that includes vmlinux.h: ./vmlinux.h:34528:3: warning: declaration does not declare anything [-Wmissing-declarations] and lays the struct out without the member, so the BTF of the object describes a struct the kernel does not have, and a program that touches the member does not build at all. Add the flag, and -Wno-microsoft-anon-tag with it, the way the selftests do. Signed-off-by: Eva Kurchatova Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260925154937.3677406-1-eva.kurchatova@virtuozzo.com --- samples/bpf/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 95a4fa1f1e44..c28e65046986 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -342,6 +342,7 @@ $(obj)/xdp_router_ipv4.bpf.o: $(obj)/xdp_sample.bpf.o $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/xdp_sample_shared.h @echo " CLANG-BPF " $@ $(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \ + -fms-extensions -Wno-microsoft-anon-tag \ -Wno-compare-distinct-pointer-types -I$(srctree)/include \ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \ From ea9358e1270ab2c3ba6f36bd9bdda68617665516 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Sat, 26 Sep 2026 13:38:35 +0200 Subject: [PATCH 334/660] selftests/bpf: Fix gcc -Wreturn-type error in the tail call subprog test gcc's C front end warns for a static function returning non-void that has no return statement at all, regardless of the for (;;) {} it ends in, and BPF CI builds test_progs-bpf_gcc with -Werror: progs/verifier_cfg.c: In function 'tail_call_or_throw_subprog': error: no return statement in function returning non-void [-Werror=return-type] Add an unreachable return after the loop. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t verifier_cfg [...] #626/21 verifier_cfg/tail call in a subprogram without an exit:OK #626 verifier_cfg:OK Summary: 1/21 PASSED, 0 SKIPPED, 0/0 FAILED Fixes: 5add2c64f474 ("selftests/bpf: Add a test for a tail call in a subprogram without an exit") Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260926113835.1193471-1-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/verifier_cfg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 07388a922e5b..6379dfc9389b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -196,6 +196,7 @@ static int tail_call_or_throw_subprog(void *ctx) bpf_tail_call(ctx, &map_array, idx); bpf_throw(0); for (;;) {} + return 0; } SEC("tc") From 61c1e63c9b651a77b1e24208b9361965c69959e6 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 28 Sep 2026 16:14:41 -0700 Subject: [PATCH 335/660] selftests/bpf: Fix linked_externs with bpf-gcc test_progs-bpf_gcc fails to build: prog_tests/linked_externs.c:17:23: error: 'struct linked_arena' has no member named 'arena' Several problems, all from bpf-gcc not supporting address_space(1). First, the arena variables are declared with __arena, but that macro is for pointers -- it marks the pointee address space. The one that carries placement is __arena_global: #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) #define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) #define __arena_global __attribute__((address_space(1))) #else #define __arena __attribute__((btf_type_tag("arena"))) #define __arena_global SEC(".addr_space.1") #endif __BPF_FEATURE_ADDR_SPACE_CAST is a clang predefine. Under clang the two are interchangeable here, since address_space(1) both places the variable and is what btf_type_tag would have described. Under bpf-gcc __arena is only a BTF type tag, so the definitions land in .data, the arena map gets no initial value, and is_skel_data() in bpftool does not emit the typed arena member -- hence the build error. Note this is distinct from skel->maps.arena, which comes from SEC(".maps") and is always present. The extern declarations need __arena_global too. Without a section, gcc does not place an extern in any DATASEC, so it stays a free-floating extern VAR rather than an entry of .addr_space.1. test_relink() only walks that DATASEC: id = btf__find_by_name_kind(btf, sec_name, BTF_KIND_DATASEC); ... ASSERT_NEQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_EXTERN, "var_resolved"); so relink_arena would report OK under bpf-gcc whether or not the linker resolved anything -- it would not be testing the allocated-section path this test exists for. Second, placement alone is not enough to run the test. Dereferencing an arena global needs an addr_space_cast, which only clang emits: clang: bpf-gcc: r1 = 0x0 ll r1 = 0x0 ll r1 = addr_space_cast(r1, 0x0, 0x1) r2 = *(u64 *)(r1 + 0x0) r1 = *(u64 *)(r1 + 0x0) so the verifier sees a scalar and rejects the program: 4: (79) r2 = *(u64 *)(r1 +0) R1 invalid mem access 'scalar' Guard the program bodies and skip the skeleton subtest, as arena_atomics.c already does. The variables stay unconditional so the arena is still populated and the skeleton still has its member. The fallback bodies reference the externs without dereferencing them: 'used' is ignored on an extern, so an unreferenced one is dropped and .addr_space.1 loses it again. With this, bpf-gcc reports: #217/1 linked_externs/skel_arena:SKIP #217/2 linked_externs/relink_arena:OK #217/3 linked_externs/relink_maps:OK with b_val present in .addr_space.1 as linkage=extern, same as clang, so relink_arena covers the same path in both. clang still passes all three. Fixes: 34354db1b148 ("selftests/bpf: Add linked_externs test for externs in allocated sections") Signed-off-by: Vineet Gupta Signed-off-by: Andrii Nakryiko Tested-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260928231441.4037097-1-vineet.gupta@linux.dev --- .../selftests/bpf/prog_tests/linked_externs.c | 15 ++++++++++-- .../selftests/bpf/progs/linked_arena1.c | 24 +++++++++++++++++-- .../selftests/bpf/progs/linked_arena2.c | 8 +++++-- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/linked_externs.c b/tools/testing/selftests/bpf/prog_tests/linked_externs.c index 29358c82dc35..ff568f5cfc42 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_externs.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_externs.c @@ -10,10 +10,20 @@ static void test_skel(void) struct linked_arena *skel; int err; - skel = linked_arena__open_and_load(); - if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + skel = linked_arena__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) return; + if (skel->data->skip_tests) { + printf("%s:SKIP: no addr_space_cast support in the BPF compiler\n", __func__); + test__skip(); + goto cleanup; + } + + err = linked_arena__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + ASSERT_EQ(skel->arena->a_val, 1, "a_val_init"); ASSERT_EQ(skel->arena->b_val, 2, "b_val_init"); @@ -28,6 +38,7 @@ static void test_skel(void) ASSERT_EQ(skel->arena->a_val, 11, "a_val"); ASSERT_EQ(skel->arena->b_val, 22, "b_val"); +cleanup: linked_arena__destroy(skel); } diff --git a/tools/testing/selftests/bpf/progs/linked_arena1.c b/tools/testing/selftests/bpf/progs/linked_arena1.c index 3e8788dec1d9..5027b4ac2887 100644 --- a/tools/testing/selftests/bpf/progs/linked_arena1.c +++ b/tools/testing/selftests/bpf/progs/linked_arena1.c @@ -10,13 +10,33 @@ struct { __uint(max_entries, 1); /* number of pages */ } arena SEC(".maps"); -long __arena a_val = 1; -extern long __arena b_val; /* defined in linked_arena2.c */ +/* + * Dereferencing an arena global needs the compiler to emit an + * addr_space_cast, which only clang does. Keep the variables so the arena + * is still populated and the skeleton still has its arena member, but skip + * the test elsewhere. + */ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST +bool skip_tests __attribute((__section__(".data"))) = false; +#else +bool skip_tests = true; +#endif + +long __arena_global a_val = 1; +extern long __arena_global b_val; /* defined in linked_arena2.c */ SEC("syscall") int sum1(void *ctx) { +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST return a_val + b_val; +#else + /* + * Reference the extern without dereferencing it, so that the relink + * test still has an extern to resolve. + */ + return (long)&b_val; +#endif } char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_arena2.c b/tools/testing/selftests/bpf/progs/linked_arena2.c index d0bcdf18a4b6..db37198854a3 100644 --- a/tools/testing/selftests/bpf/progs/linked_arena2.c +++ b/tools/testing/selftests/bpf/progs/linked_arena2.c @@ -4,15 +4,19 @@ #include #include "bpf_arena_common.h" -long __arena b_val = 2; -extern long __arena a_val; /* defined in linked_arena1.c */ +long __arena_global b_val = 2; +extern long __arena_global a_val; /* defined in linked_arena1.c */ SEC("syscall") int bump2(void *ctx) { +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST a_val += 10; b_val += 20; return a_val + b_val; +#else /* see linked_arena1.c */ + return (long)&a_val; +#endif } char _license[] SEC("license") = "GPL"; From d82229330a326beb0f1bc95dcf078df9a0359f34 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:24 -0700 Subject: [PATCH 336/660] bpf: Fix kfunc BTF parameter lookups after wide arguments Several kfunc argument checks index BTF parameters using the ABI slot number. Parameter and slot indexes diverge after a by-value argument wider than one eightbyte, so a later argument can select the wrong BTF parameter or index past the end of the function prototype. This affects the BTF lookups in check_func_arg_nullability() and check_func_arg_release(), and the parameter passed to btf_check_iter_arg() by process_iter_arg(). check_func_arg() already tracks the BTF parameter and ABI slot separately. Pass the parameter index to all three consumers while retaining argno for register and stack-slot diagnostics. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-2-ameryhung@gmail.com --- kernel/bpf/verifier.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 03dbc0e00398..b54873c8b95e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8264,12 +8264,12 @@ static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, int insn_idx, struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; - u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { @@ -8289,7 +8289,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ - btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); + btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg); if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); @@ -8920,7 +8920,7 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u } static int check_func_arg_nullability(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, argno_t argno, + struct bpf_reg_state *reg, u32 arg, argno_t argno, enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta, int insn_idx) { @@ -8933,7 +8933,7 @@ static int check_func_arg_nullability(struct bpf_verifier_env *env, if (meta->btf) { u32 arg_btf_id; - arg_btf_id = btf_params(meta->func_proto)[arg_idx_from_argno(argno)].type; + arg_btf_id = btf_params(meta->func_proto)[arg].type; expected_type = bpf_diag_fmt(env, "value of type %s", bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); } @@ -8948,7 +8948,7 @@ static int check_func_arg_nullability(struct bpf_verifier_env *env, } static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, enum bpf_arg_type arg_type, + u32 arg, argno_t argno, enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta, int insn_idx) { const char *expected_type = "pointer"; @@ -8968,7 +8968,7 @@ static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_s const struct btf_type *t; u32 ref_id; - btf_arg = &btf_params(meta->func_proto)[arg_idx_from_argno(argno)]; + btf_arg = &btf_params(meta->func_proto)[arg]; ref_id = btf_arg->type; t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); if (btf_type_is_ptr(t)) @@ -9413,7 +9413,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p return 0; } - err = check_func_arg_nullability(env, reg, argno, arg_type, meta, insn_idx); + err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx); if (err) return err; @@ -9425,7 +9425,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err) return err; - err = check_func_arg_release(env, reg, argno, arg_type, meta, insn_idx); + err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx); if (err) return err; @@ -9759,7 +9759,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); return -EINVAL; } - err = process_iter_arg(env, reg, argno, insn_idx, meta); + err = process_iter_arg(env, reg, arg, argno, insn_idx, meta); if (err < 0) return err; break; From f7be720383d3949626ef58b3dc149e1f364fa0c1 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:25 -0700 Subject: [PATCH 337/660] bpf: Identify subprog calls in argument metadata Prepare subprog calls to use the common check_func_args() path. That checker distinguishes call kinds through bpf_call_arg_meta, but btf_check_func_arg_match() leaves both BTF and the function ID zero even though it validates a program-BTF signature. Represent a subprog call with a non-NULL BTF and a zero function ID. Define helpers as NULL BTF plus nonzero ID, and kfuncs as non-NULL BTF plus nonzero ID. Requiring a nonzero kfunc ID also prevents unavailable special-kfunc IDs from matching subprog metadata. The corresponding subprog predicate is introduced later alongside its first use. Setting BTF would expose checks that treat every BTF-backed call as a kfunc. Restrict kfunc-only BTF parameter lookup, register admission, release diagnostics, no-cast alias, and projection handling to actual kfuncs. The BTF is not modified here, but bpf_call_arg_meta::btf and several downstream consumers use non-const pointers. Match those existing types instead of broadening this series with a const-correctness cleanup. No functional change is intended. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-3-ameryhung@gmail.com --- kernel/bpf/verifier.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b54873c8b95e..de9276a3ee4d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8639,18 +8639,27 @@ static bool arg_type_is_scalar(enum bpf_arg_type type) } /* - * A kfunc is named by a BTF ID, which can take the same numeric value as an - * enum bpf_func_id. Only test meta->func_id against a BPF_FUNC_* once the call - * is known to be to a helper; meta->btf is set only for a kfunc. + * A helper has no BTF and a nonzero function ID. A kfunc has both, while a + * BPF subprogram has BTF and a zero function ID. */ +static bool is_helper(const struct bpf_call_arg_meta *meta) +{ + return !meta->btf && meta->func_id; +} + static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) { - return !meta->btf && meta->func_id == func_id; + return is_helper(meta) && meta->func_id == func_id; +} + +static bool is_kfunc(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && meta->func_id; } static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) { - return meta->btf && meta->func_id == btf_id; + return is_kfunc(meta) && meta->func_id == btf_id; } static int resolve_map_arg_type(struct bpf_verifier_env *env, @@ -8963,7 +8972,7 @@ static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_s verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", meta->func_name, reg_arg_name(env, argno)); - if (meta->btf) { + if (is_kfunc(meta)) { const struct btf_param *btf_arg; const struct btf_type *t; u32 ref_id; @@ -9017,7 +9026,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; } - if (meta->btf && base_type(arg_type) == ARG_PTR_TO_BTF_ID && + if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID && (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) goto found; @@ -9040,7 +9049,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; /* Allow allocated memory for kfunc ARG_PTR_TO_MEM but not helper. */ - if (meta->btf && base_type(arg_type) == ARG_PTR_TO_MEM && + if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && type_is_ptr_alloc_obj(type)) type = PTR_TO_MEM; @@ -9363,7 +9372,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p struct bpf_call_arg_meta *meta, int insn_idx) { - const struct btf_param *btf_arg = meta->btf ? &btf_params(meta->func_proto)[arg] : NULL; + const struct btf_param *btf_arg = is_kfunc(meta) ? + &btf_params(meta->func_proto)[arg] : NULL; const struct bpf_func_proto *fn = meta->fn; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); @@ -10789,7 +10799,7 @@ err_out: } static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, - const struct btf *btf, + struct btf *btf, struct bpf_reg_state *regs) { struct bpf_subprog_info *sub = subprog_info(env, subprog); @@ -10801,8 +10811,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, u32 i; int ret, err; - /* Leave btf and func_id zero: this is neither a helper nor a kfunc. */ memset(&meta, 0, sizeof(meta)); + meta.btf = btf; meta.func_name = bpf_subprog_name(env, subprog); ret = btf_prepare_func_args(env, subprog); @@ -13783,7 +13793,7 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re * resolve types. */ if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || - (meta->btf && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, + (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, arg_btf, arg_btf_id))) strict_type_match = true; @@ -13800,7 +13810,7 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ - taking_projection = meta->btf && btf_is_projection_of(arg_tname, reg_tname); + taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), From 668a51c4ed4b235d829c38137f259da6d0eb82b1 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:26 -0700 Subject: [PATCH 338/660] bpf: Build argument prototypes for subprog calls The common argument checker consumes a parameter-indexed bpf_func_proto, while BPF subprogram argument metadata is cached by ABI slot in compact bpf_subprog_info entries. Embedding a full prototype in every fixed subprogram entry would waste memory. Embed a scratch prototype in the verifier environment. Fill it by walking BTF parameters and the cached slots with separate cursors, mirroring the kfunc representation for parameters that occupy multiple slots. Keep the compact cache as the source of truth. The verifier environment is already heap allocated, so this avoids a separate allocation, failure path, and cleanup. Keep the existing validation loop, but make it consume the generated prototype in preparation for moving BPF subprogram calls to the common argument checker. No functional change. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-4-ameryhung@gmail.com --- include/linux/bpf_verifier.h | 3 +- kernel/bpf/verifier.c | 103 ++++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 33 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c775bd757706..183759f1e014 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -976,6 +976,7 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_diag *diag; + struct bpf_func_proto bpf_subprog_scratch; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; @@ -1645,6 +1646,7 @@ struct bpf_call_arg_meta { struct btf *btf; u32 func_id; const struct bpf_func_proto *fn; + const struct btf_type *func_proto; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1657,7 +1659,6 @@ struct bpf_call_arg_meta { /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; - const struct btf_type *func_proto; const char *func_name; struct arg_constant_desc arg_constant; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index de9276a3ee4d..e677e5615fee 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9941,20 +9941,20 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p * func model recorded, or the verifier would check an argument at a slot * the JIT does not place it at. */ -static u32 kfunc_arg_slots(const struct btf_type *t) +static u32 btf_arg_slots(const struct btf_type *t) { if (btf_type_is_int(t) || btf_type_is_struct(t)) return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; return 1; } -static u32 kfunc_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto) { const struct btf_param *args = btf_params(func_proto); u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; for (i = 0; i < nargs; i++) - slots_used += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); return slots_used; } @@ -9997,7 +9997,7 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met * count. Only a proto whose parameters take a slot each can name the * argument a stack slot belongs to. */ - proto_slots = meta->btf ? kfunc_proto_slots(meta->btf, meta->func_proto) : nargs; + proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs; if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, @@ -10013,7 +10013,7 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met nslots = 1; if (args) { t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); - nslots = kfunc_arg_slots(t); + nslots = btf_arg_slots(t); } if (meta->fn->arg_type[arg] == ARG_UNUSED) @@ -10798,6 +10798,22 @@ err_out: return err; } +static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf, + const struct btf_type *func_proto, struct bpf_func_proto *proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 arg, slot = 0; + + memset(proto, 0, sizeof(*proto)); + for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { + const struct btf_type *t; + + proto->arg_type[arg] = sub->args[slot].arg_type; + t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + slot += btf_arg_slots(t); + } +} + static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf, struct bpf_reg_state *regs) @@ -10805,10 +10821,11 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; - const struct btf_param *args; + const struct btf_param *args, *stack_args; const struct btf_type *func, *func_proto; struct bpf_call_arg_meta meta; - u32 i; + struct bpf_func_proto *fn; + u32 arg, slot, nslots; int ret, err; memset(&meta, 0, sizeof(meta)); @@ -10830,33 +10847,42 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); args = btf_params(func_proto); - if (sub->arg_slot_cnt != btf_type_vlen(func_proto)) - args = NULL; + stack_args = sub->arg_slot_cnt == btf_type_vlen(func_proto) ? args : NULL; ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, - bpf_subprog_name(env, subprog), btf, args); + bpf_subprog_name(env, subprog), btf, stack_args); if (ret) return ret; + fn = &env->bpf_subprog_scratch; + gen_subprog_arg_proto(sub, btf, func_proto, fn); + meta.fn = fn; + meta.func_proto = func_proto; + /* check that BTF function arguments match actual types that the * verifier sees. */ - for (i = 0; i < sub->arg_slot_cnt; i++) { - argno_t argno = argno_from_arg(i + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - struct bpf_subprog_arg_info *arg = &sub->args[i]; + for (arg = 0, slot = 0; arg < btf_type_vlen(func_proto); arg++, slot += nslots) { + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); + enum bpf_arg_type arg_type = fn->arg_type[arg]; + argno_t argno = argno_from_arg(slot + 1); + const struct btf_type *t; + u32 k; - if (arg->arg_type == ARG_SCALAR) { + t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + nslots = btf_arg_slots(t); + + if (arg_type == ARG_SCALAR) { if (reg->type != SCALAR_VALUE) { bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; } - } else if (arg->arg_type & PTR_UNTRUSTED) { + } else if (arg_type & PTR_UNTRUSTED) { /* * Anything is allowed for untrusted arguments, as these are * read-only and probe read instructions would protect against * invalid memory access. */ - } else if (arg->arg_type == ARG_PTR_TO_CTX) { + } else if (arg_type == ARG_PTR_TO_CTX) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); if (ret < 0) return ret; @@ -10868,11 +10894,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno)); return -EINVAL; } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { + } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_MEM); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, + if (check_mem_reg(env, reg, argno, sub->args[slot].mem_size, + BPF_READ | BPF_WRITE, NULL, NULL)) return -EINVAL; /* @@ -10885,13 +10912,13 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno), subprog); return -EINVAL; } - if (!(arg->arg_type & PTR_MAYBE_NULL) && + if (!(arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { bpf_log(log, "%s is expected to be non-NULL\n", reg_arg_name(env, argno)); return -EINVAL; } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { + } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { /* * Can pass any value and the kernel won't crash, but * only PTR_TO_ARENA or SCALAR make sense. Everything @@ -10904,38 +10931,50 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno)); return -EINVAL; } - } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { + } else if (arg_type == ARG_PTR_TO_DYNPTR) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); if (ret) return ret; ret = process_dynptr_func(env, reg, argno, env->insn_idx, - arg->arg_type, &meta); + arg_type, &meta); if (ret) return ret; - } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { + } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { int err; - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { err = mark_arg_precision(env, argno); if (err) return err; continue; } - err = check_reg_type(env, reg, argno, arg->arg_type, &meta); - err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); + err = check_reg_type(env, reg, argno, arg_type, &meta); + err = err ?: check_func_arg_reg_off(env, reg, argno, arg_type); if (!err && base_type(reg->type) == PTR_TO_BTF_ID) - err = process_arg_ptr_to_btf_id(env, reg, argno, arg->arg_type, - btf_vmlinux, arg->btf_id, + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, + btf_vmlinux, sub->args[slot].btf_id, &meta, env->insn_idx); if (err) return err; } else { verifier_bug(env, "unrecognized %s type %d", - reg_arg_name(env, argno), arg->arg_type); + reg_arg_name(env, argno), arg_type); return -EFAULT; } + + for (k = 1; k < nslots; k++) { + argno_t extra_argno = argno_from_arg(slot + k + 1); + struct bpf_reg_state *extra_reg; + + extra_reg = get_func_arg_reg(caller, regs, slot + k); + if (extra_reg->type != SCALAR_VALUE) { + bpf_log(log, "%s is not a scalar\n", + reg_arg_name(env, extra_argno)); + return -EINVAL; + } + } } return 0; @@ -14511,7 +14550,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * * pointer, and neither does a slot past the last parameter. */ for (i = 0, slot = 0; i < nargs && slot < arg; i++) - slot += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); if (i >= nargs || slot != arg) return 0; @@ -18794,7 +18833,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->arg_slot_cnt = kfunc_proto_slots(meta.btf, meta.func_proto); + cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; From c40f746cc7ded326d76d1e5dd006465d8b4d7154 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:27 -0700 Subject: [PATCH 339/660] bpf: Check subprog scalar arguments in the common path BPF subprogram scalar arguments are checked separately even though the common argument checker enforces the same SCALAR_VALUE requirement. Start the migration by routing the scalar branch through check_func_arg() and update the verifier-log expectation. Use the BTF parameter index for metadata and the ABI slot index for register lookup. Route extra slots of by-value aggregates through check_arg_extra_slot() so every occupied slot follows the common path. Following patches can extend this guarded common-check branch as each remaining BPF-subprogram-specific implementation is removed. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-5-ameryhung@gmail.com --- kernel/bpf/verifier.c | 19 ++++++------------- .../selftests/bpf/progs/aggregate_arg_func.c | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e677e5615fee..e3c7c1aa0573 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10872,10 +10872,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, nslots = btf_arg_slots(t); if (arg_type == ARG_SCALAR) { - if (reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } + ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); + if (ret) + return ret; } else if (arg_type & PTR_UNTRUSTED) { /* * Anything is allowed for untrusted arguments, as these are @@ -10965,15 +10964,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, } for (k = 1; k < nslots; k++) { - argno_t extra_argno = argno_from_arg(slot + k + 1); - struct bpf_reg_state *extra_reg; - - extra_reg = get_func_arg_reg(caller, regs, slot + k); - if (extra_reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a scalar\n", - reg_arg_name(env, extra_argno)); - return -EINVAL; - } + ret = check_arg_extra_slot(env, caller, slot + k, &meta); + if (ret) + return ret; } } diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c index d0a4f84a6fbf..2132b926a8f0 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c @@ -43,7 +43,7 @@ __naked int aggregate_arg_pair_asm(void) } SEC("tc") -__failure __msg("R2 is not a scalar") +__failure __msg("R2 type=fp expected=scalar") __naked int aggregate_arg_pair_ptr_fail(void) { asm volatile ( From 3886a9fc38de0c4e03525be03570ea1473176479 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:28 -0700 Subject: [PATCH 340/660] bpf: Check global subprog untrusted arguments in the common path Global subprogram arguments tagged __arg_untrusted deliberately accept any caller value. The callee is verified separately with a read-only untrusted pointer, whose accesses use protected probe-read instructions. Keep the untrusted type in the canonical bpf_subprog_info used to prepare the callee state. Represent it as ARG_IGNORE only in the temporary bpf_func_proto used for call-site checking, then extend the guarded check_func_arg() path to cover it. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-6-ameryhung@gmail.com --- kernel/bpf/verifier.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e3c7c1aa0573..79b9e7242fee 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10806,9 +10806,18 @@ static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const stru memset(proto, 0, sizeof(*proto)); for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { + enum bpf_arg_type arg_type = sub->args[slot].arg_type; const struct btf_type *t; - proto->arg_type[arg] = sub->args[slot].arg_type; + if (arg_type & PTR_UNTRUSTED) { + /* + * An __arg_untrusted argument accepts any caller value. The + * callee treats it as read-only and uses probe-read instructions + * to protect against invalid memory access. + */ + arg_type = ARG_IGNORE; + } + proto->arg_type[arg] = arg_type; t = btf_type_skip_modifiers(btf, args[arg].type, NULL); slot += btf_arg_slots(t); } @@ -10871,16 +10880,10 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, t = btf_type_skip_modifiers(btf, args[arg].type, NULL); nslots = btf_arg_slots(t); - if (arg_type == ARG_SCALAR) { + if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) return ret; - } else if (arg_type & PTR_UNTRUSTED) { - /* - * Anything is allowed for untrusted arguments, as these are - * read-only and probe read instructions would protect against - * invalid memory access. - */ } else if (arg_type == ARG_PTR_TO_CTX) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); if (ret < 0) From 03f312c2cbd5ec0acb14f02701181af1ad0b8a08 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:29 -0700 Subject: [PATCH 341/660] bpf: Check subprog context arguments in the common path Subprog and helper/kfunc context arguments require PTR_TO_CTX with an acceptable offset. Route subprog context arguments through check_func_arg() and update the verifier-log expectation. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-7-ameryhung@gmail.com --- kernel/bpf/verifier.c | 15 ++------------- .../selftests/bpf/progs/test_global_func5.c | 2 +- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 79b9e7242fee..659820a4b805 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10880,22 +10880,11 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, t = btf_type_skip_modifiers(btf, args[arg].type, NULL); nslots = btf_arg_slots(t); - if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE) { + if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || + arg_type == ARG_PTR_TO_CTX) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) return ret; - } else if (arg_type == ARG_PTR_TO_CTX) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); - if (ret < 0) - return ret; - /* If function expects ctx type in BTF check that caller - * is passing PTR_TO_CTX. - */ - if (reg->type != PTR_TO_CTX) { - bpf_log(log, "%s expects pointer to ctx\n", - reg_arg_name(env, argno)); - return -EINVAL; - } } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_MEM); if (ret < 0) diff --git a/tools/testing/selftests/bpf/progs/test_global_func5.c b/tools/testing/selftests/bpf/progs/test_global_func5.c index 257c0569ff98..4573ecd96aa0 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func5.c +++ b/tools/testing/selftests/bpf/progs/test_global_func5.c @@ -26,7 +26,7 @@ int f3(int val, struct __sk_buff *skb) } SEC("tc") -__failure __msg("expects pointer to ctx") +__failure __msg("R2 type=fp expected=ctx") int global_func5(struct __sk_buff *skb) { return f1(skb) + f2(2, skb) + f3(3, skb); From 44cfe492d06fc4039220afeb78cf595f1c1ae76e Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:30 -0700 Subject: [PATCH 342/660] bpf: Check subprog arena arguments in the common path ARG_PTR_TO_ARENA accepts both arena pointers and scalars. The existing BPF-subprogram contract includes a constant-zero scalar; it is an arena address value rather than a conventional pointer rejected by nullability checking. Kfunc prototype generation already records this zero acceptance with PTR_MAYBE_NULL for both arena suffixes; separate JIT metadata determines whether the kernel callee receives the arena base or NULL. Apply the same call-site representation to the temporary BPF-subprogram prototype, then extend the guarded check_func_arg() path to cover arena arguments. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-8-ameryhung@gmail.com --- kernel/bpf/verifier.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 659820a4b805..efb20e50c974 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10816,6 +10816,8 @@ static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const stru * to protect against invalid memory access. */ arg_type = ARG_IGNORE; + } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { + arg_type |= PTR_MAYBE_NULL; } proto->arg_type[arg] = arg_type; t = btf_type_skip_modifiers(btf, args[arg].type, NULL); @@ -10881,7 +10883,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, nslots = btf_arg_slots(t); if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || - arg_type == ARG_PTR_TO_CTX) { + arg_type == ARG_PTR_TO_CTX || + base_type(arg_type) == ARG_PTR_TO_ARENA) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) return ret; @@ -10909,19 +10912,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno)); return -EINVAL; } - } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { - /* - * Can pass any value and the kernel won't crash, but - * only PTR_TO_ARENA or SCALAR make sense. Everything - * else is a bug in the bpf program. Point it out to - * the user at the verification time instead of - * run-time debug nightmare. - */ - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a pointer to arena or scalar.\n", - reg_arg_name(env, argno)); - return -EINVAL; - } } else if (arg_type == ARG_PTR_TO_DYNPTR) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); if (ret) From a1456ec685a609c0fd14e839728d9f8bc9c74d48 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:31 -0700 Subject: [PATCH 343/660] bpf: Check subprog dynptr arguments in the common path Subprog and helper/kfunc dynptr arguments ultimately use the same process_dynptr_func() validation. Route the subprog dynptr branch through check_func_arg() so register type, offset, and dynptr state are checked in the common order. check_func_arg() validates the register against the PTR_TO_STACK and CONST_PTR_TO_DYNPTR compatibility set before dispatching to process_dynptr_func(). Once the subprog path uses the common checker, process_dynptr_func() has no caller that bypasses this validation. Remove its now-redundant register-type check. The existing wrong-register-type test passed a NULL local to a callee that did not use the argument. Clang left the context pointer in R1, while GCC materialized zero. The common checker rejected the values at different stages and emitted different diagnostics. Obtain a PTR_TO_BTF_ID from bpf_get_current_task_btf() and keep its callee argument live. This makes both compilers exercise the intended register-type mismatch. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-9-ameryhung@gmail.com --- kernel/bpf/verifier.c | 23 +------------------ .../testing/selftests/bpf/progs/dynptr_fail.c | 11 ++++----- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index efb20e50c974..0d554e95fdb5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8133,18 +8133,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat { int spi, err = 0; - if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { - verbose(env, - "%s expected pointer to stack or const struct bpf_dynptr\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt( - env, insn_idx, argno, meta->func_name, - "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", - "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", - reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - /* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * @@ -10883,7 +10871,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, nslots = btf_arg_slots(t); if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || - arg_type == ARG_PTR_TO_CTX || + arg_type == ARG_PTR_TO_CTX || arg_type == ARG_PTR_TO_DYNPTR || base_type(arg_type) == ARG_PTR_TO_ARENA) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) @@ -10912,15 +10900,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno)); return -EINVAL; } - } else if (arg_type == ARG_PTR_TO_DYNPTR) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); - if (ret) - return ret; - - ret = process_dynptr_func(env, reg, argno, env->insn_idx, - arg_type, &meta); - if (ret) - return ret; } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { int err; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 9418dfe4d7b7..148cf4417322 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -2053,19 +2053,18 @@ __noinline long global_call_bpf_dynptr(const struct bpf_dynptr *dynptr) /* Avoid leaving this global function empty to avoid having the compiler * optimize away the call to this global function. */ + __sink(dynptr); __sink(ret); return ret; } SEC("?raw_tp") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=trusted_ptr_ expected=fp, dynptr_ptr") int test_dynptr_reg_type(void *ctx) { - struct task_struct *current = NULL; - /* R1 should be holding a PTR_TO_BTF_ID, so this shouldn't be a - * reg->type that can be passed to a function accepting a - * ARG_PTR_TO_DYNPTR | MEM_RDONLY. process_dynptr_func() should catch - * this. + struct task_struct *current = bpf_get_current_task_btf(); + /* R1 holds a PTR_TO_BTF_ID, which cannot be passed to a function + * accepting ARG_PTR_TO_DYNPTR | MEM_RDONLY. */ global_call_bpf_dynptr((const struct bpf_dynptr *)current); return 0; From 77d9384a69cdbaeaf97eca8c624c47436703e76d Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:32 -0700 Subject: [PATCH 344/660] bpf: Check global subprog BTF-ID arguments in the common path Route global ARG_PTR_TO_BTF_ID arguments through check_func_arg(). Use vmlinux BTF for their type match and retain the global-subprogram rules instead of applying kfunc-only trusted-pointer validation or runtime type resolution. The common nullability check now rejects non-nullable global BTF-ID arguments before register-type matching. Update the corresponding verifier log expectations. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-10-ameryhung@gmail.com --- kernel/bpf/verifier.c | 31 +++++-------------- .../bpf/progs/verifier_global_ptr_args.c | 4 +-- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0d554e95fdb5..8aa1336453a4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9501,7 +9501,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p case ARG_PTR_TO_BTF_ID_SOCK_COMMON: { const u32 *arg_btf_id = fn->arg_btf_id[arg]; - const struct btf *arg_btf = meta->btf ?: btf_vmlinux; + const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; if (!meta->btf) { const struct bpf_reg_types *compatible; @@ -9529,8 +9529,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p } } - if (meta->btf && (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type))) { + if (is_kfunc(meta) && (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type))) { if (!(arg_type & MEM_RCU)) { const char *actual_type, *arg_name, *expected_type; @@ -10806,6 +10806,8 @@ static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const stru arg_type = ARG_IGNORE; } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { arg_type |= PTR_MAYBE_NULL; + } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + proto->arg_btf_id[arg] = &sub->args[slot].btf_id; } proto->arg_type[arg] = arg_type; t = btf_type_skip_modifiers(btf, args[arg].type, NULL); @@ -10872,7 +10874,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || arg_type == ARG_PTR_TO_CTX || arg_type == ARG_PTR_TO_DYNPTR || - base_type(arg_type) == ARG_PTR_TO_ARENA) { + base_type(arg_type) == ARG_PTR_TO_ARENA || + base_type(arg_type) == ARG_PTR_TO_BTF_ID) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) return ret; @@ -10900,24 +10903,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, reg_arg_name(env, argno)); return -EINVAL; } - } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { - int err; - - if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - err = mark_arg_precision(env, argno); - if (err) - return err; - continue; - } - - err = check_reg_type(env, reg, argno, arg_type, &meta); - err = err ?: check_func_arg_reg_off(env, reg, argno, arg_type); - if (!err && base_type(reg->type) == PTR_TO_BTF_ID) - err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, - btf_vmlinux, sub->args[slot].btf_id, - &meta, env->insn_idx); - if (err) - return err; } else { verifier_bug(env, "unrecognized %s type %d", reg_arg_name(env, argno), arg_type); @@ -13063,7 +13048,7 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) return 0; - if (!meta->btf || arg_type_is_release(*arg_type) || + if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)]) return 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index 10019af5eb74..f639e2767e35 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -87,7 +87,7 @@ __weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted) SEC("?kprobe") __failure __log_level(2) -__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail1(void *ctx) { @@ -96,7 +96,7 @@ int trusted_task_arg_nonnull_fail1(void *ctx) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("R1 type=trusted_ptr_or_null_ expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail2(void *ctx) { From 46e0744621081c25fe3ccf4e4c270ca2844e7dff Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:33 -0700 Subject: [PATCH 345/660] bpf: Check global subprog memory arguments in the common path Route fixed-size global ARG_PTR_TO_MEM arguments through check_func_arg(). Preserve their read-write access check, nullable contract, and support for BTF-defined allocated memory. Recognize subprog calls explicitly in the common checker. Global subprog stack liveness can prove that bytes in an argument are unused by the callee. Retain the existing allowance for those poisoned stack bytes when call metadata is present, and update the nullability log expectation. The verifier also rejects packet pointers when the callee may change packet data. The concrete packet-backed register state is only available while checking the call site; the independently verified callee sees generic PTR_TO_MEM. Record this property as subprog_may_change_pkt in subprog-only call metadata and retain the packet-pointer rejection in the common fixed-memory path. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-11-ameryhung@gmail.com --- include/linux/bpf_verifier.h | 3 + kernel/bpf/verifier.c | 58 +++++++++---------- .../bpf/progs/verifier_global_ptr_args.c | 3 +- .../bpf/progs/verifier_global_subprogs.c | 2 +- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 183759f1e014..811342e3c041 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1656,6 +1656,9 @@ struct bpf_call_arg_meta { struct ret_mem_desc ret_mem; struct arg_raw_mem_desc arg_raw_mem; + /* Only set by subprog */ + bool subprog_may_change_pkt; + /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8aa1336453a4..ad18b6ab7e88 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7703,6 +7703,11 @@ size_error: return err; } +static bool is_subprog(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && !meta->func_id; +} + static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, struct bpf_call_arg_meta *meta, bool *known_memory) @@ -7721,10 +7726,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg } /* - * Only a global subprog (meta == NULL) may read poisoned stack slots: + * Only a global subprog may read poisoned stack slots: * its static stack liveness proved the callee body skips them. */ - size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + size = (is_subprog(meta) && + base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, @@ -9036,8 +9042,8 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; - /* Allow allocated memory for kfunc ARG_PTR_TO_MEM but not helper. */ - if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && + /* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */ + if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && type_is_ptr_alloc_obj(type)) type = PTR_TO_MEM; @@ -9667,6 +9673,17 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p bpf_diag_reg_type_plain(env, reg->type)); return err; } + /* + * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting + * bounds tracking information. + */ + if (is_subprog(meta) && meta->subprog_may_change_pkt && + (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) { + verbose(env, + "cannot pass packet pointer %s to %s(): function may change packet data\n", + reg_arg_name(env, argno), meta->func_name); + return -EINVAL; + } if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, arg_size, true); break; @@ -10806,6 +10823,9 @@ static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const stru arg_type = ARG_IGNORE; } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { arg_type |= PTR_MAYBE_NULL; + } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { + proto->arg_size[arg] = sub->args[slot].mem_size; + arg_type |= MEM_FIXED_SIZE | MEM_WRITE; } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { proto->arg_btf_id[arg] = &sub->args[slot].btf_id; } @@ -10821,7 +10841,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, { struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); - struct bpf_verifier_log *log = &env->log; const struct btf_param *args, *stack_args; const struct btf_type *func, *func_proto; struct bpf_call_arg_meta meta; @@ -10831,6 +10850,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, memset(&meta, 0, sizeof(meta)); meta.btf = btf; + meta.subprog_may_change_pkt = sub->changes_pkt_data; meta.func_name = bpf_subprog_name(env, subprog); ret = btf_prepare_func_args(env, subprog); @@ -10863,7 +10883,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, * verifier sees. */ for (arg = 0, slot = 0; arg < btf_type_vlen(func_proto); arg++, slot += nslots) { - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; argno_t argno = argno_from_arg(slot + 1); const struct btf_type *t; @@ -10875,34 +10894,11 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || arg_type == ARG_PTR_TO_CTX || arg_type == ARG_PTR_TO_DYNPTR || base_type(arg_type) == ARG_PTR_TO_ARENA || - base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + base_type(arg_type) == ARG_PTR_TO_BTF_ID || + base_type(arg_type) == ARG_PTR_TO_MEM) { ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); if (ret) return ret; - } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_MEM); - if (ret < 0) - return ret; - if (check_mem_reg(env, reg, argno, sub->args[slot].mem_size, - BPF_READ | BPF_WRITE, NULL, - NULL)) - return -EINVAL; - /* - * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing - * us from adjusting bounds tracking info. - */ - if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && - sub->changes_pkt_data) { - bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", - reg_arg_name(env, argno), subprog); - return -EINVAL; - } - if (!(arg_type & PTR_MAYBE_NULL) && - (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { - bpf_log(log, "%s is expected to be non-NULL\n", - reg_arg_name(env, argno)); - return -EINVAL; - } } else { verifier_bug(env, "unrecognized %s type %d", reg_arg_name(env, argno), arg_type); diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index f639e2767e35..03507eeae3cb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -389,7 +389,8 @@ __weak int subprog_pkt_ptr_changes_data(struct __sk_buff *skb __arg_ctx, SEC("?tc") __failure __log_level(2) -__msg("R2 is a packet pointer, but func#{{[0-9]+}} may change packet data") +__msg("cannot pass packet pointer R2") +__msg("function may change packet data") __msg("Caller passes invalid args into func#{{[0-9]+}} ('subprog_pkt_ptr_changes_data')") int pkt_ptr_to_global_mem_arg_changes_data(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 27fbe54e8795..574b26b5a7df 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -195,7 +195,7 @@ int arg_tag_nonnull_ptr_good(void *ctx) SEC("?raw_tp") __failure __log_level(2) -__msg("R1 is expected to be non-NULL") +__msg("Possibly NULL pointer passed to trusted R1") int arg_tag_nonnull_ptr_null_bad(void *ctx) { int y = 74; From bf82eb0183288ed4e9f045ea15de838e592709ff Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Mon, 28 Sep 2026 11:53:34 -0700 Subject: [PATCH 346/660] bpf: Check all subprog arguments in the common path All supported BPF-subprogram argument types now pass through check_func_arg() from one guarded branch in the legacy validation loop. Replace that loop and its outgoing-stack validation with check_func_args(). The scratch prototype is indexed by BTF parameter and the call metadata carries the BTF function prototype. The existing BTF argument iteration therefore maps parameters to ABI slots for both kfunc and BPF subprogram calls, including additional slots occupied by by-value aggregates. The common checker can return non-EFAULT errors other than -EINVAL, as the existing BTF-ID path already did. This is compatible with subprog call handling: only -EFAULT is fatal. Any other error marks BTF unreliable. Static subprogs can then fall back to inline verification, while global subprogs reject the call. Remove the caller-register plumbing that the dedicated loop required. Signed-off-by: Amery Hung Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928185334.1004200-12-ameryhung@gmail.com --- kernel/bpf/verifier.c | 65 ++++++++----------------------------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ad18b6ab7e88..1599bac1bac9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10835,17 +10835,13 @@ static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const stru } } -static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, - struct btf *btf, - struct bpf_reg_state *regs) +static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf) { struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); - const struct btf_param *args, *stack_args; const struct btf_type *func, *func_proto; struct bpf_call_arg_meta meta; struct bpf_func_proto *fn; - u32 arg, slot, nslots; int ret, err; memset(&meta, 0, sizeof(meta)); @@ -10867,63 +10863,24 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); - args = btf_params(func_proto); - stack_args = sub->arg_slot_cnt == btf_type_vlen(func_proto) ? args : NULL; - ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, - bpf_subprog_name(env, subprog), btf, stack_args); - if (ret) - return ret; fn = &env->bpf_subprog_scratch; gen_subprog_arg_proto(sub, btf, func_proto, fn); meta.fn = fn; meta.func_proto = func_proto; - /* check that BTF function arguments match actual types that the - * verifier sees. - */ - for (arg = 0, slot = 0; arg < btf_type_vlen(func_proto); arg++, slot += nslots) { - enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_arg(slot + 1); - const struct btf_type *t; - u32 k; - - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); - nslots = btf_arg_slots(t); - - if (arg_type == ARG_SCALAR || arg_type == ARG_IGNORE || - arg_type == ARG_PTR_TO_CTX || arg_type == ARG_PTR_TO_DYNPTR || - base_type(arg_type) == ARG_PTR_TO_ARENA || - base_type(arg_type) == ARG_PTR_TO_BTF_ID || - base_type(arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg(env, arg, slot, 0, &meta, env->insn_idx); - if (ret) - return ret; - } else { - verifier_bug(env, "unrecognized %s type %d", - reg_arg_name(env, argno), arg_type); - return -EFAULT; - } - - for (k = 1; k < nslots; k++) { - ret = check_arg_extra_slot(env, caller, slot + k, &meta); - if (ret) - return ret; - } - } - - return 0; + return check_func_args(env, &meta, env->insn_idx); } -/* Compare BTF of a function call with given bpf_reg_state. +/* + * Check that call-site argument states match a subprog's BTF signature. + * * Returns: * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. + * Other errors - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. */ -static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) +static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog) { struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; @@ -10940,7 +10897,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, if (prog->aux->func_info_aux[subprog].unreliable) return -EINVAL; - err = btf_check_func_arg_match(env, subprog, btf, regs); + err = btf_check_func_arg_match(env, subprog, btf); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. * In such cases mark the function as unreliable from BTF point of view. @@ -10959,7 +10916,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; @@ -11096,7 +11053,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EFAULT; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { @@ -11233,7 +11190,7 @@ static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, /* PTR_TO_FUNC is a pointer to a static subprog */ subprog = reg->subprogno; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; From a3df11403e8c8c3b613ee4595c1da0e3b9636971 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Sat, 26 Sep 2026 18:51:11 +0100 Subject: [PATCH 347/660] bpf: Verify BTF_KIND_LOC_PARAM vlen, flags Ensure that vlen and flags combinations are consistent for BTF_KIND_LOC_PARAMs. Fixes: 33c5a3278bdb ("btf: Extend UAPI to support BTF location (inline site) info") Suggested-by: Alexei Starovoitov Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926175113.2368566-2-alan.maguire@oracle.com --- kernel/bpf/btf.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9bcfefdfb734..8cc17a1cd25c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4823,10 +4823,53 @@ static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); return -EINVAL; } + /* All LOC_PARAMs have vlen of at least 1 */ + if (vlen < 1) + goto invalid_vlen; + + switch (p->flags) { + case BTF_LOC_PARAM_CONST: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: + if (vlen > 2) + goto invalid_vlen; + /* Ensure size/vlen are consistent */ + if (size > sizeof(__u32) && vlen != 2) + goto invalid_vlen; + if (size > sizeof(__u64)) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG: + /* Reg or reg/reg pair */ + if (vlen > 2) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF: + /* A reg deref without offset can only have vlen 1 */ + if (vlen != 1) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED: + if (vlen < 2 || vlen > 3) + goto invalid_vlen; + break; + default: + btf_verifier_log_type(env, t, "Invalid flags"); + return -EINVAL; + } btf_verifier_log_type(env, t, NULL); return meta_needed; + +invalid_vlen: + btf_verifier_log_type(env, t, "Invalid vlen"); + return -EINVAL; } static void btf_loc_param_log(struct btf_verifier_env *env, From 29f6453491f3b47893e0fc676adbff2e552b27f5 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Sat, 26 Sep 2026 18:51:12 +0100 Subject: [PATCH 348/660] bpftool: Update func representation to include function signature Augment func= output for LOCSEC entries to include a mapping from function signature to where parameters are stored; for example: [290179] LOCSEC 'inline.text' vlen=524941 func='task_pid_nr(tsk [reg0])' func_type_id=136691 loc_proto_type_id=136693 offset=2097226 func='get_current()' func_type_id=136694 loc_proto_type_id=136695 offset=2097247 func='arch_static_branch(key [address 0x2e275e8], branch [const 0x0])' func_type_id=136697 loc_proto_type_id=136700 offset=2097296 Fixes: 321562c34d5b ("bpftool: Add ability to dump LOC_PARAM, LOC_PROTO and LOCSEC") Suggested-by: Alexei Starovoitov Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20260926175113.2368566-3-alan.maguire@oracle.com --- tools/bpf/bpftool/btf.c | 123 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index e29c8a84e224..abd661db91e6 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -150,6 +150,7 @@ static void btf_loc_param_raw_str(const struct btf_loc_param *p, __u32 vlen, static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) { + const char *op = "", *prefix = ""; const struct btf_loc_param *p; __u32 i = 0, vlen; __u64 value; @@ -157,7 +158,6 @@ static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) bool negative = false; char regs[32] = {}; char num[32] = {}; - const char *op = ""; if (!t || !btf_is_loc_param(t)) { snprintf(str, sz, ""); @@ -234,8 +234,12 @@ static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) (1ULL << bits) - value; } } - snprintf(num, sizeof(num), "0x%llx%s", (unsigned long long)value, - p->flags & BTF_LOC_PARAM_ADDR ? " (addr)" : ""); + if (p->flags & BTF_LOC_PARAM_ADDR) + prefix = "address "; + else if (p->flags & BTF_LOC_PARAM_CONST) + prefix = "const "; + snprintf(num, sizeof(num), "0x%llx", + (unsigned long long)value); } if (i != vlen) { btf_loc_param_raw_str(p, vlen, str, sz); @@ -244,14 +248,111 @@ static void btf_loc_param_str(const struct btf_type *t, char *str, size_t sz) if (num[0]) op = regs[0] ? (negative ? " - " : " + ") : negative ? "-" : ""; - snprintf(str, sz, "%s%s%s%s%s", + snprintf(str, sz, "%s%s%s%s%s%s", p->flags & BTF_LOC_PARAM_DEREF ? "*(" : "", + prefix, regs, op, num, p->flags & BTF_LOC_PARAM_DEREF ? ")" : ""); } +static bool btf_locsec_append(char *str, size_t sz, size_t *off, + const char *suffix) +{ + static const char nospace_suffix[] = "..."; + bool space_left = true; + size_t left = 0, len; + + if (*off >= sz || sz < sizeof(nospace_suffix)) + return false; + left = sz - *off; + + /* + * Copy as much of the string as we can in remaining space, and + * append "..." if we would have overrun. + */ + len = strlen(suffix) + 1; + if (len > left) { + len = left; + space_left = false; + } + + memcpy(str + *off, suffix, len); + *off += len - 1; + + if (!space_left) + memcpy(str + sz - sizeof(nospace_suffix), nospace_suffix, sizeof(nospace_suffix)); + + return space_left; +} + +static void btf_locsec_func_str(const struct btf *btf, + const struct btf_loc *loc, char *str, size_t sz) +{ + const struct btf_type *func, *func_proto, *loc_proto; + const struct btf_param *params; + const __u32 *loc_params; + const char *name; + __u32 i, vlen; + size_t off = 0; + + if (!sz) + return; + + str[0] = '\0'; + func = btf__type_by_id(btf, loc->func); + if (!func || !btf_is_func(func)) + goto invalid; + + name = btf_str(btf, func->name_off); + func_proto = btf__type_by_id(btf, func->type); + loc_proto = btf__type_by_id(btf, loc->loc_proto); + if (!func_proto || !btf_is_func_proto(func_proto) || + !loc_proto || !btf_is_loc_proto(loc_proto) || + btf_vlen(func_proto) != btf_vlen(loc_proto)) + goto invalid; + + params = (const void *)(func_proto + 1); + loc_params = btf_loc_proto_params(loc_proto); + vlen = btf_vlen(func_proto); + if (!btf_locsec_append(str, sz, &off, name) || + !btf_locsec_append(str, sz, &off, "(")) + return; + for (i = 0; i < vlen; i++) { + const struct btf_type *param_loc; + char param_str[256] = {}; + + if (!params[i].type) { + /* Handle varargs func proto, must be last parameter */ + if (i != vlen - 1) + goto invalid; + if (!btf_locsec_append(str, sz, &off, i ? ", " : "") || + !btf_locsec_append(str, sz, &off, "...")) + return; + break; + } else if (loc_params[i]) { + param_loc = btf__type_by_id(btf, loc_params[i]); + btf_loc_param_str(param_loc, param_str, sizeof(param_str)); + } else { + snprintf(param_str, sizeof(param_str), ""); + } + + if (!btf_locsec_append(str, sz, &off, i ? ", " : "") || + !btf_locsec_append(str, sz, &off, + btf_str(btf, params[i].name_off)) || + !btf_locsec_append(str, sz, &off, " [") || + !btf_locsec_append(str, sz, &off, param_str) || + !btf_locsec_append(str, sz, &off, "]")) + return; + } + (void) btf_locsec_append(str, sz, &off, ")"); + return; + +invalid: + snprintf(str, sz, ""); +} + static int dump_btf_type(const struct btf *btf, __u32 id, const struct btf_type *t) { @@ -617,22 +718,26 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } for (i = 0; i < vlen; i++, locs++) { - const struct btf_type *f = btf__type_by_id(btf, locs->func); + const struct btf_type *func; const char *name = ""; + char func_str[1024] = {}; - if (f && btf_is_func(f)) - name = btf_str(btf, f->name_off); + func = btf__type_by_id(btf, locs->func); + if (func && btf_is_func(func)) + name = btf_str(btf, func->name_off); + btf_locsec_func_str(btf, locs, func_str, sizeof(func_str)); if (json_output) { jsonw_start_object(w); jsonw_uint_field(w, "func_type_id", locs->func); jsonw_string_field(w, "name", name); + jsonw_string_field(w, "func", func_str); jsonw_uint_field(w, "loc_proto_type_id", locs->loc_proto); jsonw_uint_field(w, "offset", locs->offset); jsonw_end_object(w); } else { - printf("\n\tname='%s' func_type_id=%u loc_proto_type_id=%u offset=%u", - name, locs->func, locs->loc_proto, locs->offset); + printf("\n\tfunc='%s' func_type_id=%u loc_proto_type_id=%u offset=%u", + func_str, locs->func, locs->loc_proto, locs->offset); } } if (json_output) From 3cc62f77b9749bb90bf32a85361a6bcde18ff44b Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Sat, 26 Sep 2026 18:51:13 +0100 Subject: [PATCH 349/660] selftests/bpf: Fix up bpftool btf dump test for signatures Now we show a function signature, fix up the test to expect it. Also fix the fact that we did not add the right number of parameters to the FUNC_PROTO, and ensure consts/addresses are prefixed appropriately. Fixes: 321562c34d5b ("bpftool: Add ability to dump LOC_PARAM, LOC_PROTO and LOCSEC") Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926175113.2368566-4-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/bpftool_btf_dump.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c index bed506badc75..abc62958b6ed 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -72,6 +72,7 @@ static struct btf *mk_loc_btf(void) btf__add_func_param(btf, "arg3", 1); btf__add_func_param(btf, "arg4", 1); btf__add_func_param(btf, "arg5", 1); + btf__add_func_param(btf, "arg6", 1); btf__add_func(btf, "foo", BTF_FUNC_STATIC, 2); btf__add_loc_param(btf, 4, BTF_LOC_PARAM_REG); @@ -230,28 +231,29 @@ static void test_loc_dump(const char *btf_path) { const char expected[] = "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED\n" - "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=5\n" + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=6\n" "\t'arg1' type_id=1\n" "\t'arg2' type_id=1\n" "\t'arg3' type_id=1\n" "\t'arg4' type_id=1\n" "\t'arg5' type_id=1\n" + "\t'arg6' type_id=1\n" "[3] FUNC 'foo' type_id=2 linkage=static\n" "[4] LOC_PARAM '(anon)' size=4 flags=0x8 vlen=1 values='reg1'\n" "[5] LOC_PARAM '(anon)' size=8 flags=0x38 vlen=2 values='*(reg2 + 0x10)'\n" "[6] LOC_PARAM '(anon)' size=8 flags=0x29 vlen=2 values='fbreg - 0x10'\n" - "[7] LOC_PARAM '(anon)' size=8 flags=0x6 vlen=2 values='0x123456789abcdef0 (addr)'\n" - "[8] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2 values='0xdeadbeeffeedface'\n" + "[7] LOC_PARAM '(anon)' size=8 flags=0x6 vlen=2 values='address 0x123456789abcdef0'\n" + "[8] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2 values='const 0xdeadbeeffeedface'\n" "[9] LOC_PARAM '(anon)' size=8 flags=0x39 vlen=2 values='*(fbreg - 0x20)'\n" "[10] LOC_PROTO '(anon)' vlen=6\n" "\ttype_id=4 value='reg1'\n" "\ttype_id=5 value='*(reg2 + 0x10)'\n" "\ttype_id=6 value='fbreg - 0x10'\n" - "\ttype_id=7 value='0x123456789abcdef0 (addr)'\n" - "\ttype_id=8 value='0xdeadbeeffeedface'\n" + "\ttype_id=7 value='address 0x123456789abcdef0'\n" + "\ttype_id=8 value='const 0xdeadbeeffeedface'\n" "\ttype_id=9 value='*(fbreg - 0x20)'\n" "[11] LOCSEC 'inline.text' vlen=1\n" - "\tname='foo' func_type_id=3 loc_proto_type_id=10 offset=64\n"; + "\tfunc='foo(arg1 [reg1], arg2 [*(reg2 + 0x10)], arg3 [fbreg - 0x10], arg4 [address 0x123456789abcdef0], arg5 [const 0xdeadbeeffeedface], arg6 [*(fbreg - 0x20)])' func_type_id=3 loc_proto_type_id=10 offset=64\n"; char *dump; dump = dump_raw(btf_path); From 1419795b98cf648bfff1d8030b0f1c7f5e08da4a Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 16:46:04 -0700 Subject: [PATCH 350/660] libbpf: Walk types in btf_dump_resize(), not in mark_referenced() btf_dump_mark_referenced() both walks every type added since the last resize and decides what each one references. Move the walk out to btf_dump_resize() and hand the function one type at a time, so a second per-type job can share the same pass instead of adding another one. No functional change. Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260929234608.48503-2-ihor.solodrai@linux.dev --- tools/lib/bpf/btf_dump.c | 122 +++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 62 deletions(-) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index fa995c02a170..e0293e6d3492 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -143,7 +143,7 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) va_end(args); } -static int btf_dump_mark_referenced(struct btf_dump *d); +static int btf_dump_mark_referenced(struct btf_dump *d, const struct btf_type *t); static int btf_dump_resize(struct btf_dump *d); struct btf_dump *btf_dump__new(const struct btf *btf, @@ -195,6 +195,8 @@ err: static int btf_dump_resize(struct btf_dump *d) { int err, last_id = btf__type_cnt(d->btf) - 1; + const struct btf_type *t; + __u32 i; if (last_id <= d->last_id) return 0; @@ -212,10 +214,13 @@ static int btf_dump_resize(struct btf_dump *d) d->type_states[0].emit_state = EMITTED; } - /* eagerly determine referenced types for anon enums */ - err = btf_dump_mark_referenced(d); - if (err) - return err; + for (i = d->last_id + 1; i <= last_id; i++) { + t = btf__type_by_id(d->btf, i); + + err = btf_dump_mark_referenced(d, t); + if (err) + return err; + } d->last_id = last_id; return 0; @@ -312,71 +317,64 @@ int btf_dump__dump_type(struct btf_dump *d, __u32 id) * top-level anonymous enum won't be referenced by anything, while embedded * one will. */ -static int btf_dump_mark_referenced(struct btf_dump *d) +static int btf_dump_mark_referenced(struct btf_dump *d, const struct btf_type *t) { - int i, j, n = btf__type_cnt(d->btf); - const struct btf_type *t; - __u32 vlen; + __u32 j, vlen = btf_vlen(t); - for (i = d->last_id + 1; i < n; i++) { - t = btf__type_by_id(d->btf, i); - vlen = btf_vlen(t); + switch (btf_kind(t)) { + case BTF_KIND_INT: + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + case BTF_KIND_FWD: + case BTF_KIND_FLOAT: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + break; - switch (btf_kind(t)) { - case BTF_KIND_INT: - case BTF_KIND_ENUM: - case BTF_KIND_ENUM64: - case BTF_KIND_FWD: - case BTF_KIND_FLOAT: - case BTF_KIND_LOC_PARAM: - case BTF_KIND_LOC_PROTO: - case BTF_KIND_LOCSEC: - break; + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + case BTF_KIND_PTR: + case BTF_KIND_TYPEDEF: + case BTF_KIND_FUNC: + case BTF_KIND_VAR: + case BTF_KIND_DECL_TAG: + case BTF_KIND_TYPE_TAG: + d->type_states[t->type].referenced = 1; + break; - case BTF_KIND_VOLATILE: - case BTF_KIND_CONST: - case BTF_KIND_RESTRICT: - case BTF_KIND_PTR: - case BTF_KIND_TYPEDEF: - case BTF_KIND_FUNC: - case BTF_KIND_VAR: - case BTF_KIND_DECL_TAG: - case BTF_KIND_TYPE_TAG: - d->type_states[t->type].referenced = 1; - break; + case BTF_KIND_ARRAY: { + const struct btf_array *a = btf_array(t); - case BTF_KIND_ARRAY: { - const struct btf_array *a = btf_array(t); + d->type_states[a->index_type].referenced = 1; + d->type_states[a->type].referenced = 1; + break; + } + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: { + const struct btf_member *m = btf_members(t); - d->type_states[a->index_type].referenced = 1; - d->type_states[a->type].referenced = 1; - break; - } - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: { - const struct btf_member *m = btf_members(t); + for (j = 0; j < vlen; j++, m++) + d->type_states[m->type].referenced = 1; + break; + } + case BTF_KIND_FUNC_PROTO: { + const struct btf_param *p = btf_params(t); - for (j = 0; j < vlen; j++, m++) - d->type_states[m->type].referenced = 1; - break; - } - case BTF_KIND_FUNC_PROTO: { - const struct btf_param *p = btf_params(t); + for (j = 0; j < vlen; j++, p++) + d->type_states[p->type].referenced = 1; + break; + } + case BTF_KIND_DATASEC: { + const struct btf_var_secinfo *v = btf_var_secinfos(t); - for (j = 0; j < vlen; j++, p++) - d->type_states[p->type].referenced = 1; - break; - } - case BTF_KIND_DATASEC: { - const struct btf_var_secinfo *v = btf_var_secinfos(t); - - for (j = 0; j < vlen; j++, v++) - d->type_states[v->type].referenced = 1; - break; - } - default: - return -EINVAL; - } + for (j = 0; j < vlen; j++, v++) + d->type_states[v->type].referenced = 1; + break; + } + default: + return -EINVAL; } return 0; } From 76ea312390211c03dcba741b48ef82c861ff6a4c Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 16:46:05 -0700 Subject: [PATCH 351/660] libbpf: Render decl_tags in btf_dump BTF_KIND_DECL_TAG is an attribute on a declaration: with kind_flag=0 the tag name is the argument of a btf_decl_tag(), and with kind_flag=1 it encodes a complete __attribute__. When doing btf_dump, render both forms at every declaration btf_dump emits - a record, a record member and a typedef: struct foo { ... } __attribute__((bar)); struct foo { int a __attribute__((bar)); }; typedef struct { ... } __attribute__((bar)) foo_t; A decl tag is a standalone type pointing at its target. Build an index of decl tags in btf_dump_resize(). Keep it as a flat array sorted by (target ID, tag ID). This allows for a stable emission order in btf_dump_emit_decl_tags(). btf_dump_emit_decl_tags() binary searches for where the target's entries would begin and walks tags while the target matches. A record shares its target with its members, so the component_idx is matched there. A record attribute goes after the closing brace, where __attribute__((packed)) already goes. A member attribute goes after the bit-field width: clang rejects one between the declarator and the ':'. A typedef takes it after the declarator, so a typedef of an anonymous record can carry two groups at once, one binding to the record and one to the typedef. This closes a generic gap in btf_dump, which silently dropped every decl tag it was given. Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260929234608.48503-3-ihor.solodrai@linux.dev --- tools/lib/bpf/btf_dump.c | 94 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index e0293e6d3492..996e61b5f11f 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -77,6 +77,11 @@ struct btf_dump_data { bool is_array_char; }; +struct decl_tag_desc { + __u32 target_id; + __u32 tag_id; +}; + struct btf_dump { const struct btf *btf; btf_dump_printf_fn_t printf_fn; @@ -93,6 +98,11 @@ struct btf_dump { const char **cached_names; size_t cached_names_cap; + /* decl tags, sorted by (target ID, tag ID) */ + struct decl_tag_desc *decl_tags; + size_t decl_tag_cnt; + size_t decl_tag_cap; + /* topo-sorted list of dependent type definitions */ __u32 *emit_queue; int emit_queue_cap; @@ -192,9 +202,34 @@ err: return libbpf_err_ptr(err); } +static int btf_dump_cmp_decl_tags(const void *a, const void *b) +{ + const struct decl_tag_desc *x = a, *y = b; + + if (x->target_id != y->target_id) + return x->target_id < y->target_id ? -1 : 1; + if (x->tag_id != y->tag_id) + return x->tag_id < y->tag_id ? -1 : 1; + return 0; +} + +static int btf_dump_push_decl_tag(struct btf_dump *d, __u32 id, const struct btf_type *t) +{ + if (libbpf_ensure_mem((void **)&d->decl_tags, &d->decl_tag_cap, + sizeof(*d->decl_tags), d->decl_tag_cnt + 1)) + return -ENOMEM; + + d->decl_tags[d->decl_tag_cnt++] = (struct decl_tag_desc) { + .target_id = t->type, + .tag_id = id, + }; + return 0; +} + static int btf_dump_resize(struct btf_dump *d) { int err, last_id = btf__type_cnt(d->btf) - 1; + size_t cnt = d->decl_tag_cnt; const struct btf_type *t; __u32 i; @@ -219,11 +254,25 @@ static int btf_dump_resize(struct btf_dump *d) err = btf_dump_mark_referenced(d, t); if (err) - return err; + goto out_err; + + if (btf_is_decl_tag(t)) { + err = btf_dump_push_decl_tag(d, i, t); + if (err) + goto out_err; + } } + if (d->decl_tag_cnt != cnt) + qsort(d->decl_tags, d->decl_tag_cnt, sizeof(*d->decl_tags), + btf_dump_cmp_decl_tags); + d->last_id = last_id; return 0; + +out_err: + d->decl_tag_cnt = cnt; + return err; } static void btf_dump_free_names(struct hashmap *map) @@ -256,6 +305,7 @@ void btf_dump__free(struct btf_dump *d) } } free(d->cached_names); + free(d->decl_tags); free(d->emit_queue); free(d->decl_stack); btf_dump_free_names(d->type_names); @@ -968,6 +1018,44 @@ static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, btf_dump_type_name(d, id)); } +static void btf_dump_emit_decl_tag(struct btf_dump *d, const struct btf_type *t) +{ + const char *name = btf_name_of(d, t->name_off); + + if (btf_kflag(t)) + btf_dump_printf(d, " __attribute__((%s))", name); + else + btf_dump_printf(d, " __attribute__((btf_decl_tag(\"%s\")))", name); +} + +/* + * btf_dump_resize() keeps d->decl_tags sorted by (target ID, tag ID), so the + * tags of one type form a run that binary search finds the start of, in a + * fixed order so that the same BTF always renders the same C. + */ +static void btf_dump_emit_decl_tags(struct btf_dump *d, __u32 id, int comp_idx) +{ + size_t i, lo = 0, hi = d->decl_tag_cnt, mid; + const struct btf_type *t; + + /* lower bound on target_id */ + while (lo < hi) { + mid = lo + (hi - lo) / 2; + + if (d->decl_tags[mid].target_id < id) + lo = mid + 1; + else + hi = mid; + } + + for (i = lo; i < d->decl_tag_cnt && d->decl_tags[i].target_id == id; i++) { + t = btf__type_by_id(d->btf, d->decl_tags[i].tag_id); + + if (btf_decl_tag(t)->component_idx == comp_idx) + btf_dump_emit_decl_tag(d, t); + } +} + static void btf_dump_emit_struct_def(struct btf_dump *d, __u32 id, const struct btf_type *t, @@ -1013,6 +1101,8 @@ static void btf_dump_emit_struct_def(struct btf_dump *d, prev_bitfield = false; } + /* after the bit-field width */ + btf_dump_emit_decl_tags(d, id, i); btf_dump_printf(d, ";"); } @@ -1032,6 +1122,7 @@ static void btf_dump_emit_struct_def(struct btf_dump *d, } if (packed) btf_dump_printf(d, " __attribute__((packed))"); + btf_dump_emit_decl_tags(d, id, -1); } static const char *missing_base_types[][2] = { @@ -1209,6 +1300,7 @@ static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, btf_dump_printf(d, "typedef "); btf_dump_emit_type_decl(d, t->type, name, lvl); + btf_dump_emit_decl_tags(d, id, -1); } static int btf_dump_push_decl_stack_id(struct btf_dump *d, __u32 id) From 1c36a1a41a47c6ebd8d64bd76a67e74d7a39ee19 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 16:46:06 -0700 Subject: [PATCH 352/660] selftests/bpf: Add ASSERT_TEXT_EQ() Text comparisons in test_progs repeat the same two statements: compare_text_to_expected() to print the diff, then an assert to record the failure. ASSERT_OK() on the raw return also appends a stale errno to the failure output. Add ASSERT_TEXT_EQ() after ASSERT_HAS_SUBSTR(). The assert cannot move into compare_text_to_expected() itself: the helper is linked into standalone binaries without test__fail(), and the framework unit test checks the raw mismatch return. Convert the 3 test_progs call sites plus test_ctx__dump_and_compare(), so multi-line dumps fail with a diff; prog_tests_framework.c keeps calling the helper directly. Suggested-by: Eduard Zingerman Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260929234608.48503-4-ihor.solodrai@linux.dev --- .../testing/selftests/bpf/prog_tests/bpftool_btf_dump.c | 6 ++---- tools/testing/selftests/bpf/prog_tests/btf_dump.c | 9 ++------- tools/testing/selftests/bpf/test_progs.h | 3 +++ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c index abc62958b6ed..534aa79a4278 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -207,7 +207,6 @@ static void test_dump(const char *btf_path, bool sorted) { const char *exp_path; char *dump, *exp; - int err; exp_path = sorted ? EXPECTED_SORTED : EXPECTED_UNSORTED; @@ -219,8 +218,7 @@ static void test_dump(const char *btf_path, bool sorted) if (!exp) goto out_dump; - err = compare_text_to_expected(dump, exp); - ASSERT_OK(err, sorted ? "cmp_sorted" : "cmp_unsorted"); + ASSERT_TEXT_EQ(dump, exp, sorted ? "cmp_sorted" : "cmp_unsorted"); free(exp); out_dump: @@ -260,7 +258,7 @@ static void test_loc_dump(const char *btf_path) if (!dump) return; - ASSERT_OK(compare_text_to_expected(dump, expected), "cmp_loc_dump"); + ASSERT_TEXT_EQ(dump, expected, "cmp_loc_dump"); free(dump); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index fe04a955d46c..838f7232f591 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -161,12 +161,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) goto done; } - /* - * The mismatch has already been reported, so this only has to - * register the failure. ASSERT_OK() would append a stale errno to it. - */ - err = compare_text_to_expected(dump, expected); - ASSERT_EQ(err, 0, "compare_text_to_expected"); + ASSERT_TEXT_EQ(dump, expected, "compare_text_to_expected"); done: free(expected); @@ -224,7 +219,7 @@ static void test_ctx__dump_and_compare(struct test_ctx *t, fflush(t->dump_buf_file); t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */ - ASSERT_STREQ(t->dump_buf, expected_output, message); + ASSERT_TEXT_EQ(t->dump_buf, expected_output, message); } static void test_btf_dump_incremental(void) diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index ea493c477fbd..adc18f5eceb6 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -349,6 +349,9 @@ void hexdump(const char *prefix, const void *buf, size_t len); ___ok; \ }) +#define ASSERT_TEXT_EQ(actual, expected, name) \ + ASSERT_TRUE(compare_text_to_expected(actual, expected) == 0, name) + #define ASSERT_MEMEQ(actual, expected, len, name) ({ \ static int duration = 0; \ const void *__act = actual; \ From 096ee6e38267ec620d337060d080c47fc8dfd019 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 16:46:07 -0700 Subject: [PATCH 353/660] selftests/bpf: Test btf_dump rendering of decl_tags Cover the decl tag rendering the libbpf patch added. One subtest builds a BTF with every shape that renders, plus two that must not: - a named record with both forms of tag - a typedef'd anonymous record, whose attribute precedes the declarator - a union - a record that also carries the derived packed attribute - an anonymous record inlined at a member - two records whose tags are added in reverse target order, which is what pins the sort: without it the records lose their attributes - attributes on a var and on a func, which btf_dump never declares The incremental subtest's second round adds an attribute to the new struct, pinning the later-walk indexing of appended types. A separate subtest covers members. A member attribute and a record one share a key in the index, so it checks they do not leak into each other's position, over a function pointer, an array and a bit-field - declarators the attribute has to follow rather than precede. Another covers typedefs, including a typedef of an anonymous record carrying three groups at once, binding to the member, the record and the typedef. Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260929234608.48503-5-ihor.solodrai@linux.dev --- .../selftests/bpf/prog_tests/btf_dump.c | 276 +++++++++++++++++- 1 file changed, 275 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 838f7232f591..17fa44cb9c91 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -310,6 +310,8 @@ static void test_btf_dump_incremental(void) ASSERT_OK(err, "field_ok"); err = btf__add_field(btf, "s", 6, 64, 0); ASSERT_OK(err, "field_ok"); + /* attribute on a type added after the first dump: the later walk indexes it */ + ASSERT_EQ(btf__add_decl_attr(btf, "round2", 7, -1), 8, "round2_attr"); test_ctx__dump_and_compare(&t, "struct s___2 {\n" @@ -318,7 +320,270 @@ static void test_btf_dump_incremental(void) " Y___2 = 1,\n" " } y;\n" " struct s s;\n" -"};\n\n" , "c_dump1"); +"} __attribute__((round2));\n\n", "c_dump1"); + + test_ctx__free(&t); +} + +static void test_btf_dump_decl_tags(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct s1 { int f; } __attribute__((attr1)) + * __attribute__((btf_decl_tag("plain_tag"))); + * typedef struct { int f; } __attribute__((a1)) __attribute__((a2)) t1; + * union u1 { int f; } __attribute__((uattr)); + * struct p1 { char c; int i; } __attribute__((packed)) __attribute__((pattr)); + * struct outer { struct { int g; } __attribute__((inner)) in; }; + * struct s2 { int f; } __attribute__((on_s2)); + * struct s3 { int f; } __attribute__((on_s3)); + * + * plus attributes on a var and a func, which btf_dump does not declare. + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_int(btf, "char", 1, BTF_INT_CHAR); + ASSERT_EQ(id, 2, "char_id"); + + /* a named record with one attribute of each form */ + id = btf__add_struct(btf, "s1", 4); + ASSERT_EQ(id, 3, "s1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "attr1", 3, -1), 4, "s1_attr"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "plain_tag", 3, -1), 5, "s1_plain_tag"); + + /* + * an anonymous record behind a typedef, with two attributes. This is + * the case that cannot be expressed by a caller appending to the dump: + * the attribute has to go before the declarator. + */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 6, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "a1", 6, -1), 7, "anon_attr1"); + ASSERT_EQ(btf__add_decl_attr(btf, "a2", 6, -1), 8, "anon_attr2"); + id = btf__add_typedef(btf, "t1", 6); + ASSERT_EQ(id, 9, "typedef_id"); + + /* unions are records too */ + id = btf__add_union(btf, "u1", 4); + ASSERT_EQ(id, 10, "u1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "u1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "uattr", 10, -1), 11, "u1_attr"); + + /* composes with the packed attribute, which is derived from layout */ + id = btf__add_struct(btf, "p1", 5); + ASSERT_EQ(id, 12, "p1_id"); + ASSERT_OK(btf__add_field(btf, "c", 2, 0, 0), "p1_field_c"); + ASSERT_OK(btf__add_field(btf, "i", 1, 8, 0), "p1_field_i"); + ASSERT_EQ(btf__add_decl_attr(btf, "pattr", 12, -1), 13, "p1_attr"); + + /* an anonymous record inlined at a member, the third emission position */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 14, "inner_id"); + ASSERT_OK(btf__add_field(btf, "g", 1, 0, 0), "inner_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "inner", 14, -1), 15, "inner_attr"); + id = btf__add_struct(btf, "outer", 4); + ASSERT_EQ(id, 16, "outer_id"); + ASSERT_OK(btf__add_field(btf, "in", 14, 0, 0), "outer_field"); + + /* not rendered: a var has no declaration in the C output */ + id = btf__add_var(btf, "v", BTF_VAR_GLOBAL_ALLOCATED, 3); + ASSERT_EQ(id, 17, "var_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "var_attr", 17, -1), 18, "var_attr"); + /* not rendered: neither does a func */ + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 19, "proto_id"); + id = btf__add_func(btf, "fn", BTF_FUNC_GLOBAL, 19); + ASSERT_EQ(id, 20, "func_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "func_attr", 20, -1), 21, "func_attr"); + + /* + * Tags added in reverse target order: the tag with the lower type ID + * names the record with the higher one, so the index is in emission + * order only once btf_dump_resize() has sorted it. Without the sort + * the lookups land outside their runs and the records lose their + * attributes. + */ + id = btf__add_struct(btf, "s2", 4); + ASSERT_EQ(id, 22, "s2_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s2_field"); + id = btf__add_struct(btf, "s3", 4); + ASSERT_EQ(id, 23, "s3_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s3_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s3", 23, -1), 24, "s3_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s2", 22, -1), 25, "s2_attr"); + + test_ctx__dump_and_compare(&t, +"struct s1 {\n" +" int f;\n" +"} __attribute__((attr1)) __attribute__((btf_decl_tag(\"plain_tag\")));\n" +"\n" +"typedef struct {\n" +" int f;\n" +"} __attribute__((a1)) __attribute__((a2)) t1;\n" +"\n" +"union u1 {\n" +" int f;\n" +"} __attribute__((uattr));\n" +"\n" +"struct p1 {\n" +" char c;\n" +" int i;\n" +"} __attribute__((packed)) __attribute__((pattr));\n" +"\n" +"struct outer {\n" +" struct {\n" +" int g;\n" +" } __attribute__((inner)) in;\n" +"};\n" +"\n" +"struct s2 {\n" +" int f;\n" +"} __attribute__((on_s2));\n" +"\n" +"struct s3 {\n" +" int f;\n" +"} __attribute__((on_s3));\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * An attribute on a member is emitted at the member declaration, and one on + * the record at the closing brace. Both are keyed by the record in the index, + * so this also checks that they do not leak into each other's position. + */ +static void test_btf_dump_decl_tags_members(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct m1 { + * int (*fp)(void) __attribute__((fp_attr)); + * int a; + * int b __attribute__((b1)) __attribute__((b2)); + * int arr[4] __attribute__((arr_attr)); + * } __attribute__((rec)); + * struct bits { int x : 3 __attribute__((x_attr)); int y : 29; }; + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_array(btf, 1, 1, 4); + ASSERT_EQ(id, 2, "array_id"); + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 3, "proto_id"); + id = btf__add_ptr(btf, 3); + ASSERT_EQ(id, 4, "ptr_id"); + + id = btf__add_struct(btf, "m1", 32); + ASSERT_EQ(id, 5, "m1_id"); + ASSERT_OK(btf__add_field(btf, "fp", 4, 0, 0), "m1_fp"); + ASSERT_OK(btf__add_field(btf, "a", 1, 64, 0), "m1_a"); + ASSERT_OK(btf__add_field(btf, "b", 1, 96, 0), "m1_b"); + ASSERT_OK(btf__add_field(btf, "arr", 2, 128, 0), "m1_arr"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec", 5, -1), 6, "m1_rec_attr"); + /* the attribute follows the whole declarator, pointer and array alike */ + ASSERT_EQ(btf__add_decl_attr(btf, "fp_attr", 5, 0), 7, "m1_fp_attr"); + /* two on one member, to pin the ordering */ + ASSERT_EQ(btf__add_decl_attr(btf, "b1", 5, 2), 8, "m1_b1"); + ASSERT_EQ(btf__add_decl_attr(btf, "b2", 5, 2), 9, "m1_b2"); + ASSERT_EQ(btf__add_decl_attr(btf, "arr_attr", 5, 3), 10, "m1_arr_attr"); + + /* a bit-field: the attribute has to follow the width */ + id = btf__add_struct(btf, "bits", 4); + ASSERT_EQ(id, 11, "bits_id"); + ASSERT_OK(btf__add_field(btf, "x", 1, 0, 3), "bits_x"); + ASSERT_OK(btf__add_field(btf, "y", 1, 3, 29), "bits_y"); + ASSERT_EQ(btf__add_decl_attr(btf, "x_attr", 11, 0), 12, "bits_x_attr"); + + test_ctx__dump_and_compare(&t, +"struct m1 {\n" +" int (*fp)(void) __attribute__((fp_attr));\n" +" int a;\n" +" int b __attribute__((b1)) __attribute__((b2));\n" +" int arr[4] __attribute__((arr_attr));\n" +"} __attribute__((rec));\n" +"\n" +"struct bits {\n" +" int x: 3 __attribute__((x_attr));\n" +" int y: 29;\n" +"};\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * A typedef is the third and last kind btf_dump declares. Its attributes go + * after the declarator, so a typedef of an anonymous record can carry three + * groups in one declaration, binding to three different entities. + */ +static void test_btf_dump_decl_tags_typedef(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * typedef int td __attribute__((t1)) + * __attribute__((btf_decl_tag("t2"))); + * typedef struct { + * int f __attribute__((memb_attr)); + * } __attribute__((rec_attr)) both __attribute__((both_attr)); + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_typedef(btf, "td", 1); + ASSERT_EQ(id, 2, "td_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "t1", 2, -1), 3, "td_attr1"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "t2", 2, -1), 4, "td_attr2"); + + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 5, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec_attr", 5, -1), 6, "anon_rec_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "memb_attr", 5, 0), 7, "anon_memb_attr"); + id = btf__add_typedef(btf, "both", 5); + ASSERT_EQ(id, 8, "both_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "both_attr", 8, -1), 9, "both_attr"); + + test_ctx__dump_and_compare(&t, +"typedef int td __attribute__((t1)) __attribute__((btf_decl_tag(\"t2\")));\n" +"\n" +"typedef struct {\n" +" int f __attribute__((memb_attr));\n" +"} __attribute__((rec_attr)) both __attribute__((both_attr));\n" +"\n", "dump_and_compare"); test_ctx__free(&t); } @@ -1104,6 +1369,15 @@ void test_btf_dump() { if (test__start_subtest("btf_dump: incremental")) test_btf_dump_incremental(); + if (test__start_subtest("btf_dump: decl_tags")) + test_btf_dump_decl_tags(); + + if (test__start_subtest("btf_dump: decl_tags_members")) + test_btf_dump_decl_tags_members(); + + if (test__start_subtest("btf_dump: decl_tags_typedef")) + test_btf_dump_decl_tags_typedef(); + if (test__start_subtest("btf_dump: type_tags")) test_btf_dump_type_tags(); From 2c964f6c5d8eb4319ec9ba3b79f99a8c77bdb564 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 16:46:08 -0700 Subject: [PATCH 354/660] selftests/bpf: Show the tag kflag in the raw BTF dump helper A kflag on BTF_KIND_DECL_TAG or BTF_KIND_TYPE_TAG means the tag string is an __attribute__ list rather than a btf_decl_tag()/btf_type_tag() argument. fprintf_btf_type_raw() did not print it, so a tag added with btf__add_decl_tag() and an attribute added with btf__add_decl_attr() dumped identically apart from their type IDs. Print " kflag=1" for those two kinds, only when the flag is set, so no existing expected string changes. Add a btf_write subtest for btf__add_decl_attr() and btf__add_type_attr(). Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260929234608.48503-6-ihor.solodrai@linux.dev --- tools/testing/selftests/bpf/btf_helpers.c | 10 ++++-- .../selftests/bpf/prog_tests/btf_write.c | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c index 85a4d15d1dd1..e04b61c08cd8 100644 --- a/tools/testing/selftests/bpf/btf_helpers.c +++ b/tools/testing/selftests/bpf/btf_helpers.c @@ -114,9 +114,12 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_TYPEDEF: - case BTF_KIND_TYPE_TAG: fprintf(out, " type_id=%u", t->type); break; + case BTF_KIND_TYPE_TAG: + fprintf(out, " type_id=%u%s", t->type, + btf_kflag(t) ? " kflag=1" : ""); + break; case BTF_KIND_ARRAY: { const struct btf_array *arr = btf_array(t); @@ -203,8 +206,9 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) fprintf(out, " size=%u", t->size); break; case BTF_KIND_DECL_TAG: - fprintf(out, " type_id=%u component_idx=%d", - t->type, btf_decl_tag(t)->component_idx); + fprintf(out, " type_id=%u component_idx=%d%s", + t->type, btf_decl_tag(t)->component_idx, + btf_kflag(t) ? " kflag=1" : ""); break; case BTF_KIND_LOC_PARAM: { struct btf_loc_param *p = btf_loc_param(t); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index 5c84723cf254..3bed3a28c06a 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -606,6 +606,38 @@ cleanup: btf__free(base); } +static void test_btf_attrs(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + btf__add_decl_tag(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "member_attr", 2, 0); + btf__add_type_tag(btf, "tag", 1); + btf__add_type_attr(btf, "tag", 1); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0", + "[3] DECL_TAG 'tag' type_id=2 component_idx=-1", + "[4] DECL_TAG 'tag' type_id=2 component_idx=-1 kflag=1", + "[5] DECL_TAG 'member_attr' type_id=2 component_idx=0 kflag=1", + "[6] TYPE_TAG 'tag' type_id=1", + "[7] TYPE_TAG 'tag' type_id=1 kflag=1"); + + btf__free(btf); +} + void test_btf_write() { if (test__start_subtest("btf_add")) @@ -614,4 +646,6 @@ void test_btf_write() test_btf_add_btf(); if (test__start_subtest("btf_add_btf_split")) test_btf_add_btf_split(); + if (test__start_subtest("btf_attrs")) + test_btf_attrs(); } From 99c3e74f8d9695b57bda22a8f2e0ad220dcb78bc Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Tue, 29 Sep 2026 19:08:51 -0700 Subject: [PATCH 355/660] selftests/bpf: Skip callx switch table test for clang-22 with cpu=v4 clang-22 miscompiles call_by_switch() when targeting -mcpu=v4: it turns the switch into an array of pointers to functions and then erroneously removes the array in BPFAsmPrinter::doFinalization(), breaking the build with: error: Undefined temporary symbol .Lswitch.table.call_by_switch Fixed by llvm commit 9fc458a1d95b ("[BPF] Fix erroneous removal of non-jump-table globals"), first released in clang-23; older clangs are fine. test_progs-cpuv4 is built by default whenever clang supports v4, so selftests/bpf fails to build with any clang-22 release. Guard just this test with the compiler/cpu check and leave a __skip placeholder for the broken combination, so the gap is reported as SKIP instead of silently missing. Fixes: b018e998e645 ("selftests/bpf: Add tests for callx through pointers in read-only data") Signed-off-by: Ihor Solodrai Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930020851.71691-1-ihor.solodrai@linux.dev --- .../bpf/progs/verifier_callx_rodata.c | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c index db3169a1ddf6..3e7a1c2c2a9a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -803,6 +803,14 @@ int callx_c_ops_selected(void *ctx) return use_shape(ops, 3); } +/* + * clang-22 with -mcpu=v4 drops the lookup table it builds for this switch + * ("Undefined temporary symbol .Lswitch.table.call_by_switch"). + * Fixed in clang-23 by llvm commit 9fc458a1d95b. + * + * https://github.com/llvm/llvm-project/commit/9fc458a1d95b + */ +#if __clang_major__ != 22 || __BPF_CPU_VERSION__ < 4 /* the compiler might turn the switch into a table that has no symbol */ static __noinline int call_by_switch(unsigned int idx, int x) { @@ -847,6 +855,18 @@ int callx_c_switch_table(void *ctx) return call_by_switch(op_idx - 8, 3); } +#else + +SEC("socket") +__skip("clang-22 bug with -mcpu=v4, switch table test cannot be built") +__success +int callx_c_switch_table(void *ctx) +{ + return 0; +} + +#endif + /* * Misaligned pointers to functions are ignored, the rest of the data is * accessible as before. The program doesn't have callx, so libbpf doesn't From acff58e305175df35985082b0e79103a2497f702 Mon Sep 17 00:00:00 2001 From: Luis Vieira Date: Tue, 29 Sep 2026 22:56:12 -0300 Subject: [PATCH 356/660] libbpf: Fix cleanup on invalid CO-RE relocation offsets bpf_object__relocate_core() allocates a CO-RE candidate cache before processing relocation records. Two instruction offset validation failures return -EINVAL directly instead of going through the common cleanup path. These checks were moved into bpf_object__relocate_core() when bpf_core_apply_relo() was split, preserving direct returns that were safe in the previous helper but now bypass cleanup. Route both failures through the out path so the candidate cache and target BTF override are released. Signed-off-by: Luis Vieira Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260929-libbpf-core-relo-cleanup-v1-1-c9cd0b5775a9@gmail.com --- tools/lib/bpf/libbpf.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 238e35c1ee3f..fdd69aac39bd 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -6255,8 +6255,11 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { - if (rec->insn_off % BPF_INSN_SZ) - return -EINVAL; + if (rec->insn_off % BPF_INSN_SZ) { + err = -EINVAL; + goto out; + } + insn_idx = rec->insn_off / BPF_INSN_SZ; prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); if (!prog) { @@ -6283,8 +6286,10 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) * relocated, so it's enough to just subtract in-section offset */ insn_idx = insn_idx - prog->sec_insn_off; - if (insn_idx >= prog->insns_cnt) - return -EINVAL; + if (insn_idx >= prog->insns_cnt) { + err = -EINVAL; + goto out; + } insn = &prog->insns[insn_idx]; if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) { From 8295510622f35cf5721d412ec7d3c223f929ebea Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 26 Sep 2026 15:30:43 +0200 Subject: [PATCH 357/660] selftests/bpf: Test non-sleepable kfunc context A sleepable kfunc call can be rejected because the program itself is not sleepable or because a sleepable program is inside an active critical section. Exercise the two causes separately. Call a sleepable kfunc directly from a non-sleepable tracing program, then call it from a sleepable tracing program inside an RCU read-side critical section. Assert both the legacy verifier messages and the structured reasons. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/2e42a1a2bf45f4d2aba7495bdc9f147558055740e2f3c8b9dae255f6c57fc13c@mail.kernel.org/ Link: https://lore.kernel.org/r/20260924170646.2366016-6-memxor@gmail.com Link: https://lore.kernel.org/bpf/20260926133048.2962553-2-memxor@gmail.com --- .../selftests/bpf/progs/preempt_lock.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 81c459435680..955391da326a 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c @@ -6,6 +6,8 @@ #include "bpf_experimental.h" extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; SEC("?tc") __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") @@ -179,6 +181,30 @@ int preempt_sleepable_kfunc(void *ctx) return 0; } +SEC("?fentry/" SYS_PREFIX "sys_getpgid") +__failure __msg("program must be sleepable to call sleepable kfunc bpf_copy_from_user_str") +__msg("cannot be used in non-sleepable program") +int non_sleepable_kfunc(void *ctx) +{ + u32 data; + + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("kernel func bpf_copy_from_user_str is sleepable within rcu_read_lock region") +__msg("cannot be used in RCU read lock region") +int sleepable_kfunc_in_rcu(void *ctx) +{ + u32 data; + + bpf_rcu_read_lock(); + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + bpf_rcu_read_unlock(); + return 0; +} + int __noinline preempt_global_subprog(void) { preempt_balance_subprog(); From d91052063bee29106cd51ba45761425ec3210063 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 26 Sep 2026 15:30:44 +0200 Subject: [PATCH 358/660] bpf: Correct Program Structure diagnostic context Program Structure reports have two attribution gaps. A missing jump table is reported at the beginning of its subprogram rather than at the gotox that needs the table, and the subprogram-layout checks run before func_info and line_info are validated and installed, so their reports cannot name the source function or line. Report the missing jump table at the gotox instruction that looks it up, rather than at the start of its subprogram. func_info and line_info validation only needs the subprogram boundaries found by add_subprogs() and the LD_ABS and tail-call properties that check_subprogs() collects while scanning instructions; it does not depend on the layout checks themselves. Split the property collection into its own nested subprogram and instruction scan, together with the program's callx marker, and run bpf_check_btf_info() before check_subprogs(). The jump-boundary and fallthrough reports then carry validated source information without changing either check. CO-RE relocations are unaffected: commit c26e97721b17 ("bpf: Apply CO-RE relocations before subprogram validation") applies them before add_subprogs(), and they stay there. Only the func_info and line_info validation moves. Fixes: a8f427835394 ("bpf: Report Program Structure CFG errors") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/cf2f420c2b21de440a7dc51b1565c0f06d4b539640ee5c03384e5d77bcfb5686@mail.kernel.org/ Link: https://lore.kernel.org/bpf/20260926133048.2962553-3-memxor@gmail.com --- kernel/bpf/cfg.c | 2 +- kernel/bpf/verifier.c | 54 +++++++++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index b0bd9ba951df..d8a579680e5b 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -393,7 +393,7 @@ subprog_jt(int t, struct bpf_verifier_env *env) if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); bpf_diag_program_structure( - env, subprog_start, "missing jump table", + env, t, "missing jump table", "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", "No jump table was found for the subprogram that starts at instruction %u.", subprog_start); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1599bac1bac9..b840b3eb9b22 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3107,6 +3107,34 @@ static int add_kfuncs(struct bpf_verifier_env *env) return 0; } +static void find_subprog_properties(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog = env->subprog_info; + struct bpf_insn *insn = env->prog->insnsi; + int cur_subprog; + + for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { + int i; + + for (i = subprog[cur_subprog].start; + i < subprog[cur_subprog + 1].start; i++) { + u8 code = insn[i].code; + + if (code == (BPF_JMP | BPF_CALL) && + insn[i].src_reg == 0 && + insn[i].imm == BPF_FUNC_tail_call) { + subprog[cur_subprog].has_tail_call = true; + subprog[cur_subprog].tail_call_reachable = true; + } + if (BPF_CLASS(code) == BPF_LD && + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) + subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; + } + } +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -3120,17 +3148,6 @@ static int check_subprogs(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code; - if (code == (BPF_JMP | BPF_CALL) && - insn[i].src_reg == 0 && - insn[i].imm == BPF_FUNC_tail_call) { - subprog[cur_subprog].has_tail_call = true; - subprog[cur_subprog].tail_call_reachable = true; - } - if (BPF_CLASS(code) == BPF_LD && - (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) - subprog[cur_subprog].has_ld_abs = true; - if (bpf_is_callx(&insn[i])) - env->has_callx = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3154,9 +3171,10 @@ static int check_subprogs(struct bpf_verifier_env *env) } next: if (i == subprog_end - 1) { - /* to avoid fall-through from one subprog into another + /* + * To avoid fall-through from one subprog into another, * the last insn of the subprog should be either exit - * or unconditional jump back or bpf_throw call + * or unconditional jump back or bpf_throw call. */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && @@ -22511,17 +22529,19 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Discover all subprograms before validating their layout and BTF. */ + /* Discover all subprograms and collect the properties needed by BTF validation. */ ret = add_subprogs(env); if (ret < 0) goto skip_full_check; - ret = check_subprogs(env); + find_subprog_properties(env); + + /* Validate BTF before reporting subprogram layout errors. */ + ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout. */ - ret = bpf_check_btf_info(env, attr, uattr); + ret = check_subprogs(env); if (ret < 0) goto skip_full_check; From 917cfd1ed5ed295a0ad3df5deb72d2ae9c796dbd Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 26 Sep 2026 15:30:45 +0200 Subject: [PATCH 359/660] selftests/bpf: Test Program Structure diagnostic context Exercise each corrected Program Structure report. Place a missing-table gotox after another instruction so its attribution differs from the subprogram start. Add malformed subprogram layouts for a branch crossing a subprogram boundary and a subprogram that falls through its end. Both cases carry BTF line records and assert that their structured reports include the corresponding source function and file. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/cf2f420c2b21de440a7dc51b1565c0f06d4b539640ee5c03384e5d77bcfb5686@mail.kernel.org/ Link: https://lore.kernel.org/r/20260924170646.2366016-8-memxor@gmail.com Link: https://lore.kernel.org/bpf/20260926133048.2962553-4-memxor@gmail.com --- .../selftests/bpf/progs/verifier_cfg.c | 40 +++++++++++++++++++ .../selftests/bpf/progs/verifier_gotox.c | 2 + 2 files changed, 42 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 6379dfc9389b..b429fa7a08f5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -56,6 +56,46 @@ __naked void out_of_range_jump2(void) " ::: __clobber_all); } +static __naked __noinline __used int cross_subprog_target(void) +{ + asm volatile (" \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("jump across subprogram boundary") +__failure __msg("jump out of range from insn 1") +__msg("jump_across_subprog_boundary @ verifier_cfg.c") +__naked void jump_across_subprog_boundary(void) +{ + asm volatile (" \ + call cross_subprog_target; \ + goto +1; \ + exit; \ +" ::: __clobber_all); +} + +static __naked __noinline __used int fallthrough_subprog(void) +{ + asm volatile (" \ + r0 = 0; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("subprogram fallthrough") +__failure __msg("last insn is not an exit or jmp") +__msg("fallthrough_subprog @ verifier_cfg.c") +__naked void subprog_fallthrough(void) +{ + asm volatile (" \ + call fallthrough_subprog; \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("invalid DW LDSX instruction in diagnostics") __failure __msg("BUG_ldx_99") diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index f5a9878c7b8d..dc7baa0f2458 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -101,9 +101,11 @@ __naked void jump_table_terminal_gotox_subprog(void) */ SEC("socket") __failure __msg("no jump tables found for subprog starting at 0") +__msg(">>> 1 | (0d) gotox r0") __naked void jump_table_no_jump_table(void) { asm volatile (" \ + r0 = 0; \ .8byte %[gotox_r0]; \ r0 = 1; \ exit; \ From 6a75c73eebd4d497ded7d08b47894f9ddbebb5a9 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 26 Sep 2026 17:45:14 +0200 Subject: [PATCH 360/660] bpftool: Exit from the stream wait signal handler with _exit() bpftool prog tracelog -w ends from its SIGINT, SIGHUP and SIGTERM handler, which calls exit(). exit() is not async-signal-safe: it runs atexit handlers and flushes stdio streams, and the signal may land while the read loop is inside fwrite() or fflush() on the stream being flushed. Whether that deadlocks or flushes a half-updated buffer depends on the C library. Call _exit() instead. The loop flushes every chunk as soon as it is printed, so the stdio buffer is empty except while a chunk is being written, and skipping the exit-time flush loses nothing. While here, note in the manual page that opening a stream as a file descriptor arrives in Linux 7.4, and restrict the -w/--wait description to the stdout/stderr form of the tracelog command, since the trace pipe form does not take the option. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Acked-by: Quentin Monnet Link: https://lore.kernel.org/bpf/d64c6402-175f-4ea6-be6d-1a544a95c97c@qmon.net Link: https://lore.kernel.org/bpf/20260926154515.191689-1-memxor@gmail.com --- tools/bpf/bpftool/Documentation/bpftool-prog.rst | 8 +++++--- tools/bpf/bpftool/prog.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 17aa0d560e32..8dbd34ef3d8b 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -190,7 +190,8 @@ bpftool prog tracelog { stdout | stderr } *PROG* By default, bpftool prints the output buffered so far and exits. With **-w** or **--wait**, it keeps printing new output as the program produces it, until the program is unloaded or is hit. Waiting requires a - kernel that supports opening a stream as a file descriptor. + kernel that supports opening a stream as a file descriptor, introduced + upstream in Linux 7.4. bpftool prog run *PROG* data_in *FILE* [data_out *FILE* [data_size_out *L*]] [ctx_in *FILE* [ctx_out *FILE* [ctx_size_out *M*]]] [repeat *N*] Run BPF program *PROG* in the kernel testing infrastructure for BPF, @@ -274,8 +275,9 @@ OPTIONS signing. -w, --wait - When dumping a program stream with **bpftool prog tracelog**, wait for new - output instead of exiting once the buffered output has been printed. + When dumping a program stream with **bpftool prog tracelog** { **stdout** | + **stderr** } *PROG*, wait for new output instead of exiting once the + buffered output has been printed. EXAMPLES ======== diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 5e27b22444f2..a60414aaa494 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1121,7 +1121,7 @@ enum prog_tracelog_mode { static void exit_stream(int signo) { - exit(0); + _exit(0); } /* Consumes prog_fd. */ From b5a4aa31abd6fe90009b63e35dc18c67d041ec0c Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Thu, 1 Oct 2026 07:19:01 -0700 Subject: [PATCH 361/660] bpf, cgroup: Fix cgroup struct_ops query for a second attach type Two things in __cgroup_bpf_query() work only because CGROUP_TCP_SOCK_OPS is the only struct_ops attach type. It calls cgroup_bpf_enabled(atype) with an atype that find_atype_by_struct_ops_id() works out at runtime. That macro is an asm goto and needs a constant. Today the compiler can see there is only one value; add a second type and the build breaks with "impossible constraint in 'asm'". The check is not needed anyway. When the key is off nothing is attached, so progs[atype] and effective[atype] are empty and the count loop leaves total_cnt at 0. The other attach types already use that loop with no such check. Drop the check and the skip_count label. And find_atype_by_struct_ops_id() matches on type_id alone. An attach type whose subsystem is not built keeps type_id 0, so a query for type 0 finds it and returns success with nothing instead of -ENOENT. Skip such slots. Fixes: 369d9dcd8fb8 ("bpf: Add infrastructure to support attaching struct_ops to cgroups") Signed-off-by: Shakeel Butt Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Yafang Shao Acked-by: Tejun Heo Link: https://patch.msgid.link/20261001141901.3225830-1-shakeel.butt@linux.dev --- kernel/bpf/cgroup.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 2bbe77de89f0..ce82963bcfc7 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -45,6 +45,7 @@ static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id) for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) { if (cgroup_bpf_is_struct_ops_atype(atype) && + cgroup_struct_ops[atype].type_id && cgroup_struct_ops[atype].type_id == type_id) return atype; } @@ -1448,8 +1449,6 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, return -ENOENT; from_atype = to_atype = atype; flags = 0; - if (!cgroup_bpf_enabled(atype)) - goto skip_count; } else if (type == BPF_LSM_CGROUP) { if (!effective_query && attr->query.prog_cnt && prog_ids && !prog_attach_flags) @@ -1476,7 +1475,6 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } } -skip_count: /* always output uattr->query.attach_flags as 0 during effective query */ flags = effective_query ? 0 : flags; if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) From 11de29f38a3455e5785c5fd5f38d93c413c5e822 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:12 +0000 Subject: [PATCH 362/660] bpf: Use an llist for page allocations bpf_map_alloc_pages() does not use its map argument. Storing allocated pages in an array also forces arena callers to allocate a separate pointer array. Expose the single-page allocator as bpf_alloc_page(), rename the bulk helper to bpf_alloc_pages(), and return bulk allocations through an llist using page->pcp_llist. Add bpf_free_pages() to safely release all pages remaining on such a list. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-2-emil@etsalapatis.com --- include/linux/bpf.h | 6 ++- kernel/bpf/arena.c | 95 +++++++++++++++++++------------------------- kernel/bpf/syscall.c | 39 ++++++++++-------- 3 files changed, 67 insertions(+), 73 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 4bae3796c42f..904b539810c7 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2902,8 +2902,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index c6369ea5e208..de4f7c7f68f5 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct bpf_arena *arena; - struct page **pages; + struct llist_head *pages; int i; }; @@ -158,13 +158,17 @@ struct clear_range_data { static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; + struct llist_node *node; struct page *page; pte_t pteval; if (!data) return 0; - page = d->pages[d->i]; + node = READ_ONCE(d->pages->first); + if (WARN_ON_ONCE(!node)) + return -EINVAL; + page = llist_entry(node, struct page, pcp_llist); /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; @@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif + WARN_ON_ONCE(llist_del_first(d->pages) != node); d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; @@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); - if (err) + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE); + if (!arena->scratch_page) goto err_free_arena; range_tree_init(&arena->rt); @@ -481,7 +486,9 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; + LLIST_HEAD(pages); struct page *page, *new_page = NULL; + struct apply_range_data data; vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; @@ -543,8 +550,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) * The probed page was freed meanwhile or preallocation failed; * try the non-blocking allocator, we cannot sleep here. */ - ret = bpf_map_alloc_pages(map, map->numa_node, 1, &new_page); - if (ret) { + new_page = bpf_alloc_page(map->numa_node); + if (!new_page) { fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; } @@ -555,12 +562,14 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; } - struct apply_range_data data = { - .arena = arena, .pages = &new_page, .i = 0 - }; + llist_add(&new_page->pcp_llist, &pages); + data.arena = arena; + data.pages = &pages; + data.i = 0; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { + llist_del_first(&pages); range_tree_set(&arena->rt, vmf->pgoff, 1); fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; @@ -716,13 +725,12 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); struct mem_cgroup *new_memcg, *old_memcg; struct apply_range_data data; - struct page **pages = NULL; - long remaining, mapped = 0; - long alloc_pages; + LLIST_HEAD(pages); + long mapped = 0; unsigned long flags; long pgoff = 0; u32 uaddr32; - int ret, i; + int ret; if (node_id != NUMA_NO_NODE && ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) @@ -741,15 +749,9 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ - alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *)); - pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!pages) { - bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; - } data.arena = arena; - data.pages = pages; + data.pages = &pages; + data.i = 0; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; @@ -767,45 +769,28 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt if (ret) goto out_unlock_free_pages; - remaining = page_cnt; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - while (remaining) { - long this_batch = min(remaining, alloc_pages); + ret = bpf_alloc_pages(node_id, page_cnt, &pages); + if (ret) + goto out; - /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */ - memset(pages, 0, this_batch * sizeof(struct page *)); + /* + * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 + * will not overflow 32-bit. Lower 32-bit need to represent + * contiguous user address range. + * Map these pages at kern_vm_start base. + * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow + * lower 32-bit and it's ok. + */ + ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32, + page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); + mapped = data.i; + if (ret) + goto out; - ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages); - if (ret) - goto out; - - /* - * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 - * will not overflow 32-bit. Lower 32-bit need to represent - * contiguous user address range. - * Map these pages at kern_vm_start base. - * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow - * lower 32-bit and it's ok. - */ - data.i = 0; - ret = apply_to_page_range(&init_mm, - kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT), - this_batch << PAGE_SHIFT, apply_range_set_cb, &data); - if (ret) { - /* data.i pages were mapped, account them and free the remaining */ - mapped += data.i; - for (i = data.i; i < this_batch; i++) - free_pages_nolock(pages[i], 0); - goto out; - } - - mapped += this_batch; - remaining -= this_batch; - } flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - kfree_nolock(pages); bpf_map_memcg_exit(old_memcg, new_memcg); return clear_lo32(arena->user_vm_start) + uaddr32; out: @@ -819,7 +804,7 @@ out: out_unlock_free_pages: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); out_free_pages: - kfree_nolock(pages); + bpf_free_pages(&pages); bpf_map_memcg_exit(old_memcg, new_memcg); return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ac52f4ae414c..a5df15a6cd51 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -602,7 +602,7 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -static struct page *__bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid) { if (!can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); @@ -613,27 +613,34 @@ static struct page *__bpf_alloc_page(int nid) 0); } -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **pages) +void bpf_free_pages(struct llist_head *pages) { - unsigned long i, j; + struct llist_node *node; + struct page *page, *tmp; + + node = llist_del_all(pages); + llist_for_each_entry_safe(page, tmp, node, pcp_llist) + free_pages_nolock(page, 0); +} + +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages) +{ + unsigned long i; struct page *pg; - int ret = 0; for (i = 0; i < nr_pages; i++) { - pg = __bpf_alloc_page(nid); - - if (pg) { - pages[i] = pg; - continue; - } - for (j = 0; j < i; j++) - free_pages_nolock(pages[j], 0); - ret = -ENOMEM; - break; + pg = bpf_alloc_page(nid); + if (!pg) + goto free_pages; + llist_add(&pg->pcp_llist, pages); } - return ret; + return 0; + +free_pages: + bpf_free_pages(pages); + return -ENOMEM; } static int btf_field_cmp(const void *a, const void *b) From cbe7d2bf7355416815c878e839d03028f4724dfb Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:13 +0000 Subject: [PATCH 363/660] bpf: Add sleepable argument to bpf_alloc_pages() bpf_alloc_pages() currently decides whether it may use the blocking page allocator from the current execution context alone. Let callers further restrict that choice by passing whether their context is sleepable. Use the blocking allocator only when both the caller and runtime context allow sleeping. Add __GFP_RETRY_MAYFAIL so this path can reclaim without invoking the OOM killer when the allocation is charged to another memcg. Existing non-sleepable callers retain the no-lock allocation behavior. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-3-emil@etsalapatis.com --- include/linux/bpf.h | 4 ++-- kernel/bpf/arena.c | 6 +++--- kernel/bpf/syscall.c | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 904b539810c7..670eb9f3f20a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2902,9 +2902,9 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -struct page *bpf_alloc_page(int nid); +struct page *bpf_alloc_page(int nid, bool sleepable); int bpf_alloc_pages(int nid, unsigned long nr_pages, - struct llist_head *pages); + struct llist_head *pages, bool sleepable); void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index de4f7c7f68f5..c556df7730c4 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -317,7 +317,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE); + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true); if (!arena->scratch_page) goto err_free_arena; @@ -550,7 +550,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) * The probed page was freed meanwhile or preallocation failed; * try the non-blocking allocator, we cannot sleep here. */ - new_page = bpf_alloc_page(map->numa_node); + new_page = bpf_alloc_page(map->numa_node, false); if (!new_page) { fault_ret = VM_FAULT_SIGBUS; goto out_err_locked_memcg; @@ -771,7 +771,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - ret = bpf_alloc_pages(node_id, page_cnt, &pages); + ret = bpf_alloc_pages(node_id, page_cnt, &pages, false); if (ret) goto out; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a5df15a6cd51..ef8fb2f6e6e3 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -602,14 +602,14 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -struct page *bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid, bool sleepable) { - if (!can_alloc_pages()) + if (!sleepable || !can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); return alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT - | __GFP_NOWARN, + | __GFP_NOWARN | __GFP_RETRY_MAYFAIL, 0); } @@ -624,13 +624,13 @@ void bpf_free_pages(struct llist_head *pages) } int bpf_alloc_pages(int nid, unsigned long nr_pages, - struct llist_head *pages) + struct llist_head *pages, bool sleepable) { unsigned long i; struct page *pg; for (i = 0; i < nr_pages; i++) { - pg = bpf_alloc_page(nid); + pg = bpf_alloc_page(nid, sleepable); if (!pg) goto free_pages; llist_add(&pg->pcp_llist, pages); From 5316ede1f5d07653a669285ca715a24152d729c6 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:14 +0000 Subject: [PATCH 364/660] bpf: Add sleepable arena page allocation path The bpf_arena_alloc_pages() function currently only allocates pages inside a spinlock critical section with IRQs off. This forces the use of alloc_pages_nolock() in the BPF allocator, even when the caller is a sleepable BPF function. This in turn causes allocation failures even in cases where falling into the allocator slow path and possibly sleeping would eventually succeed. This can be triggered consistently by heavy BPF arena users like scx. Allocate the arena pages before taking the critical section and pass whether the caller can sleep to bpf_alloc_pages(). This lets sleepable callers use the blocking allocator while non-sleepable callers retain the no-lock allocation behavior. Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe") Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-4-emil@etsalapatis.com --- kernel/bpf/arena.c | 73 +++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index c556df7730c4..0ff707707da0 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -713,6 +713,27 @@ static u64 clear_lo32(u64 val) return val & ~(u64)~0U; } +static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff) +{ + int ret; + + /* Special case where user is requesting specific range. */ + if (uaddr) { + ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt); + if (ret) + return ret; + return range_tree_clear(&arena->rt, *pgoff, page_cnt); + } + + ret = range_tree_find(&arena->rt, page_cnt); + if (ret < 0) + return ret; + + *pgoff = ret; + + return range_tree_clear(&arena->rt, *pgoff, page_cnt); +} + /* * Allocate pages and vmap them into kernel vmalloc area. * Later the pages will be mmaped into user space vma. @@ -730,6 +751,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt unsigned long flags; long pgoff = 0; u32 uaddr32; + long addr = 0; int ret; if (node_id != NUMA_NO_NODE && @@ -747,8 +769,12 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt /* requested address will be outside of user VMA */ return 0; } - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable); + if (ret) + goto out_memcg; + data.arena = arena; data.pages = &pages; data.i = 0; @@ -756,25 +782,14 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; - if (uaddr) { - ret = is_range_tree_set(&arena->rt, pgoff, page_cnt); - if (ret) - goto out_unlock_free_pages; - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); - } else { - ret = pgoff = range_tree_find(&arena->rt, page_cnt); - if (pgoff >= 0) - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); + ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + goto out_free_pages; } - if (ret) - goto out_unlock_free_pages; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - ret = bpf_alloc_pages(node_id, page_cnt, &pages, false); - if (ret) - goto out; - /* * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 * will not overflow 32-bit. Lower 32-bit need to represent @@ -787,26 +802,30 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); mapped = data.i; if (ret) - goto out; + goto out_unmap; flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - bpf_map_memcg_exit(old_memcg, new_memcg); - return clear_lo32(arena->user_vm_start) + uaddr32; -out: + + addr = clear_lo32(arena->user_vm_start) + uaddr32; + goto out_memcg; + +out_unmap: + /* Error handling: Undo partial mappings. */ + flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - if (mapped) { - flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + if (mapped) arena_free_pages(arena, uaddr32, mapped, sleepable); - } - goto out_free_pages; -out_unlock_free_pages: - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + out_free_pages: + /* Error handling: Free back any unmapped pages. */ bpf_free_pages(&pages); + +out_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; + + return addr; } /* From 0f6512272538b1419ccb3a3ea192bde882edc511 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:15 +0000 Subject: [PATCH 365/660] selftests/bpf: Test large allocations for both sleepable/nonsleepable arena users We have relaxed the limitation of 1024-page batches for both sleepable and nonsleepable arena allocations by removing the intermediate page array. Add a test to confirm that large allocations succeed for both. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-5-emil@etsalapatis.com --- .../bpf/progs/verifier_arena_large.c | 64 +++++++++++++++---- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 6ab8730d4878..dbdea14ca76f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c @@ -10,6 +10,9 @@ #include #define ARENA_SIZE (1ull << 32) +#define LARGE_PAGE_CNT 1025 + +volatile int zero = 0; struct { __uint(type, BPF_MAP_TYPE_ARENA); @@ -284,6 +287,7 @@ int big_alloc2(void *ctx) return 0; } +/* Nonsleepable because it binds to a socket program. */ SEC("socket") __success __retval(0) int big_alloc3(void *ctx) @@ -291,24 +295,56 @@ int big_alloc3(void *ctx) #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) char __arena *pages; u64 i; + int err = 0; - /* - * Allocate 2051 pages in one go to check how kmalloc_nolock() handles large requests. - * Since kmalloc_nolock() can allocate up to 1024 struct page * at a time, this call should - * result in three batches: two batches of 1024 pages each, followed by a final batch of 3 - * pages. - */ - pages = bpf_arena_alloc_pages(&arena, NULL, 2051, NUMA_NO_NODE, 0); + /* Verify that a nonsleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); if (!pages) - return 0; + return 1; - bpf_for(i, 0, 2051) - pages[i * PAGE_SIZE] = 123; - bpf_for(i, 0, 2051) - if (pages[i * PAGE_SIZE] != 123) - return i; + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; - bpf_arena_free_pages(&arena, pages, 2051); + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; +#endif + return 0; +} + +/* SYSCALL programs are always sleepable. */ +SEC("syscall") +__success __retval(0) +int big_alloc4(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + char __arena *pages; + u64 i; + int err = 0; + + /* Verify that a sleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); + if (!pages) + return 1; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; #endif return 0; } From baf4e66f038fb1181dd0b1085c6b0913399c289f Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:16 +0000 Subject: [PATCH 366/660] bpf: Directly store kfunc desc index in instruction off field Currently, the verifier sort the kfunc desc table twice: Once during kfunc collection by function ID, useful during verification, and once by immediate address, useful during JIT bytecode lowering time. The latter sort can be removed by storing in the instruction the kfunc desc array index the desc is in and using that instead. Modify the JITs to follow the same convention. This change simplifies the subsequent patch that adds per-call site function specialization. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-6-emil@etsalapatis.com --- include/linux/bpf.h | 4 +-- include/linux/bpf_verifier.h | 7 ++-- kernel/bpf/fixups.c | 70 +++++------------------------------- kernel/bpf/verifier.c | 25 ++++++++++--- 4 files changed, 33 insertions(+), 73 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 670eb9f3f20a..eed7f8f1e417 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3301,7 +3301,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3644,7 +3644,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 811342e3c041..b309ff7db493 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1788,12 +1788,12 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +static_assert(MAX_KFUNC_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; @@ -1801,9 +1801,8 @@ struct bpf_kfunc_desc { struct bpf_kfunc_desc_tab { u32 nr_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. * * Grown one entry at a time by bpf_add_kfunc_call(). */ diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 37cf130ebb57..cb7219ff68bd 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -117,73 +115,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) return dst_reg; } -static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) -{ - const struct bpf_kfunc_desc *d0 = a; - const struct bpf_kfunc_desc *d1 = b; - - if (d0->imm != d1->imm) - return d0->imm < d1->imm ? -1 : 1; - if (d0->offset != d1->offset) - return d0->offset < d1->offset ? -1 : 1; - return 0; -} - const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { - const struct bpf_kfunc_desc desc = { - .imm = insn->imm, - .offset = insn->off, - }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - res = bsearch(&desc, tab->descs, tab->nr_descs, - sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); - - return res ? &res->func_model : NULL; -} - -static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) -{ - unsigned long call_imm; + if (insn->off < 0 || insn->off >= tab->nr_descs) + return NULL; + res = &tab->descs[insn->off]; if (bpf_jit_supports_far_kfunc_call()) { - call_imm = desc->func_id; - } else { - call_imm = BPF_CALL_IMM(desc->addr); - /* Check whether the relative offset overflows desc->imm */ - if ((unsigned long)(s32)call_imm != call_imm) { - verbose(env, "address of kernel func_id %u is out of range\n", - desc->func_id); - return -EINVAL; - } - } - desc->imm = call_imm; - return 0; -} - -static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) -{ - struct bpf_kfunc_desc_tab *tab; - int i, err; - - tab = env->prog->aux->kfunc_tab; - if (!tab) - return 0; - - for (i = 0; i < tab->nr_descs; i++) { - err = set_kfunc_desc_imm(env, &tab->descs[i]); - if (err) - return err; + if (res->func_id != insn->imm) + return NULL; + } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { + return NULL; } - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), - kfunc_desc_cmp_by_imm_off, NULL); - return 0; + return &res->func_model; } static int add_kfunc_in_insns(struct bpf_verifier_env *env, @@ -2720,10 +2671,6 @@ next_insn: } } - ret = sort_kfunc_descs_by_imm_off(env); - if (ret) - return ret; - return 0; } @@ -2897,4 +2844,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } - diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b840b3eb9b22..f5ff812259c6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2584,12 +2584,16 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) } int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { + struct bpf_kfunc_desc_tab *tab; const struct bpf_kfunc_desc *desc; - desc = find_kfunc_desc(prog, func_id, btf_fd_idx); - if (!desc) + tab = prog->aux->kfunc_tab; + if (desc_idx >= tab->nr_descs) + return -EFAULT; + desc = &tab->descs[desc_idx]; + if (desc->func_id != func_id) return -EFAULT; *func_addr = (u8 *)desc->addr; @@ -22038,6 +22042,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { struct bpf_kfunc_desc *desc; + unsigned long call_imm; + u16 desc_idx; int err; if (!insn->imm) { @@ -22057,13 +22063,22 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } + desc_idx = desc - env->prog->aux->kfunc_tab->descs; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (!bpf_jit_supports_far_kfunc_call()) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + } + insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; From 23290213e1abfc71731a6491ca278141e8cbbf3b Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:17 +0000 Subject: [PATCH 367/660] bpf: Support per-call-site kfunc specialization specialize_kfunc() currently updates the canonical kfunc descriptor in place. Different call sites therefore cannot select different specializations of the same kfunc, and the selected target depends on verification order. Keep canonical descriptors unchanged for verifier lookups. Specialize a copy for each call site, then reuse or append an immutable target descriptor and store its index in the finalized call instruction. Allow space for one canonical and one specialized target per kfunc. This is conservative because most kfuncs do not specialize. Adding specialized kfunc versions does not require resorting the array because lookups are only used for deduplication and func_model lookup. Deduplication for specialized kfuncs is handled by the specialization process itself, while all specializations of a function share the same proto and func_model. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-7-emil@etsalapatis.com --- include/linux/bpf_verifier.h | 12 +++-- kernel/bpf/verifier.c | 88 +++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b309ff7db493..79c4be5c33f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1788,7 +1788,9 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 -static_assert(MAX_KFUNC_DESCS <= S16_MAX + 1); +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; @@ -1800,11 +1802,15 @@ struct bpf_kfunc_desc { struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during * verification. JITs use the descriptor index stored in the finalized - * call's off field. + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f5ff812259c6..ba0dfefe7209 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2579,7 +2579,7 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } @@ -2933,10 +2933,15 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (tab->nr_descs != tab->nr_base_descs) { + verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", + tab->nr_descs, tab->nr_base_descs); + return -EFAULT; + } err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2994,7 +2999,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -22021,6 +22027,66 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc, + u16 base_desc_idx, u16 *desc_idx) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + + if (base_desc_idx >= tab->nr_base_descs) { + verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", + base_desc_idx, tab->nr_base_descs); + return -EFAULT; + } + if (tab->descs[base_desc_idx].func_id != desc.func_id) { + verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); + return -EFAULT; + } + if (tab->descs[base_desc_idx].offset != desc.offset) { + verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); + return -EFAULT; + } + + if (tab->descs[base_desc_idx].addr == desc.addr) { + *desc_idx = base_desc_idx; + return 0; + } + + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) { + *desc_idx = i; + return 0; + } + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + *desc_idx = tab->nr_descs; + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -22041,9 +22107,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; unsigned long call_imm; - u16 desc_idx; + u16 base_desc_idx, desc_idx; + bool near_call; int err; if (!insn->imm) { @@ -22063,13 +22131,17 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } - desc_idx = desc - env->prog->aux->kfunc_tab->descs; + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; + + near_call = !bpf_jit_supports_far_kfunc_call(); + desc_copy = *desc; + desc = &desc_copy; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) { + if (near_call) { call_imm = BPF_CALL_IMM(desc->addr); if ((unsigned long)(s32)call_imm != call_imm) { verbose(env, "address of kernel func_id %u is out of range\n", @@ -22078,6 +22150,10 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } insn->imm = call_imm; } + + err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); + if (err) + return err; insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { From fc38f07781ca3e40f4f39532d83b5cb3d75a3ab5 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Fri, 2 Oct 2026 10:52:18 +0000 Subject: [PATCH 368/660] selftests/bpf: Test per-call site function specialization Add a test to ensure function call specialization is done per-call site. Use bpf_dynptr_from_file that has observably different behavior between its sleepable and nonsleepable versions. The sleepable path fails in the sleepable __kernel_read() call with -EIO, while the nonsleepable fails in the page-cache lookup path with -EFAULT. Test that whatever the order the nonsleepable and sleepable calls are made in the program, both call sites use the correct specialized kfunc. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261002105218.6171-8-emil@etsalapatis.com --- .../selftests/bpf/prog_tests/file_reader.c | 15 ++ .../testing/selftests/bpf/progs/file_reader.c | 129 ++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 48aae7ea0e4b..e59c6c87e9d5 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c @@ -7,10 +7,12 @@ #include "file_reader_fail.skel.h" #include #include +#include const char *user_ptr = "hello world"; char file_contents[256000]; void *addr; +__u64 beyond_eof_offset; void *get_executable_base_addr(void) { @@ -26,12 +28,18 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { + struct stat st; int fd, page_sz = sysconf(_SC_PAGESIZE); ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) return 1; + if (!ASSERT_OK(fstat(fd, &st), "fstat /proc/self/exe")) { + close(fd); + return 1; + } + beyond_eof_offset = st.st_size + (1ULL << 30); do { cur = read(fd, file_contents + n, sizeof(file_contents) - n); @@ -75,6 +83,7 @@ static void run_test(const char *prog_name) memcpy(skel->bss->user_buf, file_contents, sizeof(file_contents)); skel->bss->pid = getpid(); + skel->bss->beyond_eof_offset = beyond_eof_offset; err = file_reader__load(skel); if (!ASSERT_OK(err, "file_reader__load")) @@ -110,6 +119,12 @@ void test_file_reader(void) if (test__start_subtest("on_open_validate_file_read")) run_test("on_open_validate_file_read"); + if (test__start_subtest("on_open_non_sleepable_first")) + run_test("on_open_non_sleepable_first"); + + if (test__start_subtest("on_open_sleepable_first")) + run_test("on_open_sleepable_first"); + if (test__start_subtest("negative")) RUN_TESTS(file_reader_fail); } diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index aa2c05cce2b3..8b972fd26d73 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c @@ -27,9 +27,14 @@ char tmp_buf[256000]; int pid = 0; int err, run_success = 0; +__u64 beyond_eof_offset; static int validate_file_read(struct file *file); static int task_work_callback(struct bpf_map *map, void *key, void *value); +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value); + +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; SEC("lsm/file_open") int on_open_expect_fault(void *c) @@ -81,6 +86,101 @@ int on_open_validate_file_read(void *c) return 0; } +/* + * Exercise bpf_dynptr_from_file() first from a non-sleepable LSM program and + * then from its sleepable task-work callback. Reading beyond EOF makes the two + * backing implementations return different errors. + */ +SEC("lsm/file_open") +int on_open_non_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct elem *work; + struct file *file; + int key = 0; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 1; + return 0; + } + + /* The non-sleepable reader cannot fault in an uncached folio. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EFAULT) { + err = 2; + return 0; + } + + work = bpf_map_lookup_elem(&arrmap, &key); + if (!work) { + err = 3; + return 0; + } + + ret = bpf_task_work_schedule_signal(task, &work->tw, &arrmap, + sleepable_second_callback); + if (ret) + err = 4; + return 0; +} + +/* + * Exercise the opposite fixup order: the first call is made from a sleepable + * LSM program, while the RCU read-side section makes the second non-sleepable. + */ +SEC("lsm.s/file_open") +int on_open_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 7; + return 0; + } + + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EIO) { + err = 8; + goto out; + } + + bpf_rcu_read_lock(); + ret = bpf_dynptr_from_file(file, 0, &dynptr); + bpf_rcu_read_unlock(); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EFAULT) { + err = 9; + goto out; + } + + run_success = 1; +out: + bpf_put_file(file); + return 0; +} + /* Called in a sleepable context, read 256K bytes, cross check with user space read data */ static int task_work_callback(struct bpf_map *map, void *key, void *value) { @@ -97,6 +197,35 @@ static int task_work_callback(struct bpf_map *map, void *key, void *value) return 0; } +/* Task-work callbacks are verified as sleepable. */ +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 5; + return 0; + } + + /* freader_fetch() converts __kernel_read()'s short read at EOF to -EIO. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EIO) { + err = 6; + return 0; + } + + run_success = 1; + return 0; +} + static int verify_dynptr_read(struct bpf_dynptr *ptr, u32 off, char *user_buf, u32 len) { int i; From 5ec398d9ec8fe13f8058af9a2e5f5e8f46e0a35a Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 28 Sep 2026 17:11:21 +0800 Subject: [PATCH 369/660] bpf: Roll back freplace link state when bpf_arch_text_poke() fails A freplace attach claims the target prog by bumping tgt_prog->aux->freplace_link_cnt and setting tr->extension_prog. bpf_arch_text_poke() then makes the extension take effect. If the poke fails, the claims are never released: the attach unwinds through bpf_link_cleanup(), which clears link->prog, so bpf_trampoline_unlink_prog() never runs. Drop the link count under ext_mutex on the error path, and set tr->extension_prog only after the poke succeeded. The count is still bumped before the poke: it blocks prog_array updates while the entry is patched. Fixes: d6083f040d5d ("bpf: Prevent tailcall infinite loop caused by freplace") Fixes: be8704ff07d2 ("bpf: Introduce dynamic program extensions") Suggested-by: Leon Hwang Signed-off-by: Yuan Chen Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Link: https://patch.msgid.link/20260928091121.219333-1-chenyuan_fl@163.com --- kernel/bpf/trampoline.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index d880ad6913b3..fc94d6062109 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1026,10 +1026,17 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; + err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + node->link->prog->bpf_func); + if (err) { + /* Undo the claim from bpf_freplace_link_tgt_prog(). */ + guard(mutex)(&tgt_prog->aux->ext_mutex); + tgt_prog->aux->freplace_link_cnt--; + return err; + } tr->extension_prog = node->link->prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - node->link->prog->bpf_func); + return 0; } err = bpf_trampoline_add_prog(tr, node, cnt); if (err) From 72c0681c0e75aa4fe6ce46ac0bb63fb5be9132d7 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:00 +0000 Subject: [PATCH 370/660] bpf: Allow bitwise ops, shifts and mul/div on pointers with CAP_PERFMON Rust's core::fmt keeps a flag in the low bit of a pointer: r2 = *(u64 *)(r1 + 0) r3 = r2 r3 &= 1 r2 >>= 1 The verifier rejects it: r0 &= 8 R0 bitwise operator &= on pointer prohibited Negation and byte swap of a pointer already produce a number when allow_ptr_leaks is set. Do the same for bitwise ops, shifts, *=, /= and %=, 64-bit and 32-bit. The result is an unknown number. With CAP_PERFMON the program can store the pointer and load it back as a number already. += and -= are not changed: they keep the pointer, 32-bit += is rejected. Pointers that allow no arithmetic and pointers that may be NULL are still rejected. Two tests in verifier_value_illegal_alu store through the result of &= and /=. They are rejected at the store now. Update expected messages. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 13 ++++++++++++- .../bpf/progs/verifier_value_illegal_alu.c | 8 ++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c1fc00eb8487..866c0540c68a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15571,8 +15571,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn u32 dst = insn->dst_reg; const char *reason; int ret, bounds_ret; + bool to_scalar; dst_reg = ®s[dst]; + /* + * Only += and -= keep a pointer. Any other op makes a number of it, + * which is fine when the program may see values of pointers anyway. + */ + to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { @@ -15583,7 +15589,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return 0; } - if (BPF_CLASS(insn->code) != BPF_ALU64) { + if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); @@ -15649,6 +15655,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EACCES; } + if (to_scalar) { + __mark_reg_unknown(env, dst_reg); + return 0; + } + /* For 'scalar += pointer', dst_reg inherits the complete pointer * register state. Individual fields may be adjusted later by pointer * arithmetic. Callers guarantee that below does not overwrite off_reg. diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 4d8273c258d5..9f669cf85c59 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c @@ -22,8 +22,8 @@ struct { SEC("socket") __description("map element value illegal alu op, 1") -__failure __msg("R0 bitwise operator &= on pointer") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator &= on pointer") __naked void value_illegal_alu_op_1(void) { asm volatile (" \ @@ -70,8 +70,8 @@ l0_%=: exit; \ SEC("socket") __description("map element value illegal alu op, 3") -__failure __msg("R0 pointer arithmetic with /= operator") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 pointer arithmetic with /= operator") __naked void value_illegal_alu_op_3(void) { asm volatile (" \ From 7ba7c5b3989ed28617399a47fc70c292ff606b23 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:01 +0000 Subject: [PATCH 371/660] selftests/bpf: Add tests for ALU on pointers with CAP_PERFMON Check that &=, |=, ^=, >>=, *= and 32-bit &= on a pointer to map value produce a number for a program with CAP_PERFMON and that the number can't be dereferenced. They are rejected without CAP_PERFMON, with CAP_BPF or without it. Pointer that may be NULL and pointer to map are rejected as before. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-3-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/progs/verifier_value_illegal_alu.c | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 9f669cf85c59..31663338866d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c @@ -165,6 +165,171 @@ __naked void map_ptr_illegal_alu_op(void) : __clobber_all); } +SEC("socket") +__description("tag in the low bit of a pointer, and, shift") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__naked void ptr_tag_and_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r2 = r0; \ + r2 >>= 1; \ + r3 = r0; \ + r3 |= 1; \ + r3 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("tag in the low bit of a pointer, CAP_BPF without CAP_PERFMON") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__caps_unpriv(CAP_BPF) +__naked void ptr_tag_cap_bpf(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("number op= pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 pointer arithmetic with *= operator") +__naked void number_mul_ptr(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = 7; \ + r1 *= r0; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("pointer with the tag cleared is a number") +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator |= on pointer") +__naked void ptr_tag_cleared_deref(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r0 |= 1; \ + r0 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("shift of a pointer that may be NULL") +__failure __msg("R0 pointer arithmetic on map_value_or_null prohibited, null-check it first") +__failure_unpriv +__naked void ptr_or_null_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 >>= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("and of a pointer to map") +__failure __msg("R0 pointer arithmetic on map_ptr prohibited") +__failure_unpriv +__naked void map_ptr_and(void) +{ + asm volatile (" \ + r0 = %[map_hash_48b] ll; \ + r0 &= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("32-bit and of a pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R0 32-bit pointer arithmetic prohibited") +__naked void ptr_and32(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + w0 &= 1; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + SEC("flow_dissector") __description("flow_keys illegal alu op with variable offset") __failure __msg("R7 pointer arithmetic on flow_keys prohibited") From 4c651a91bdfcded7a775a177d8918a4127555c78 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:02 +0000 Subject: [PATCH 372/660] bpf: Treat load and store through a number as arena access There are no address spaces in Rust. LLVM emits plain loads and stores for arena memory, without cast_kern: r3 = 0x20 ll // R_BPF_64_64 .bss, libbpf puts it into arena r2 = *(u64 *)(r3 + 0) // R3 invalid mem access 'scalar' lock *(u64 *)(r2 + 8) += r1 Add BPF_F_ARENA_SCALAR flag of BPF_PROG_LOAD. When the program is loaded with it and has an arena treat ldx, stx, st and atomics through a number as arena access. It's as safe as access through PTR_TO_ARENA: JIT adds the base of arena to the low 32 bits of the address. The program has an arena only with CAP_BPF and CAP_PERFMON. Registers of the program are not changed, since Rust compares and stores the address after the access. bpf_do_misc_fixups() copies the low 32 bits into BPF_REG_AX and the access goes through it: r2 = *(u64 *)(r3 + 0) -> w12 = w3 r2 = *(u64 *)(r12 + 0) Constant blinding needs BPF_REG_AX for immediates and leaves insns that use BPF_REG_AX alone. So the immediate of st through a number is not blinded, the rest of the program is: *(u64 *)(r3 + 0) = 1 -> w12 = w3 *(u64 *)(r12 + 0) = 1 The same insn may see PTR_TO_ARENA on another path. It works for both. It's a flag, since a number is also what the verifier makes of a pointer that went away when the program has CAP_PERFMON: a ringbuf record after bpf_ringbuf_submit(), a pointer to the packet after bpf_skb_pull_data(). Programs in C that have an arena keep "invalid mem access 'scalar'" for such bugs. JIT tells with bpf_jit_supports_arena_scalar() that it takes BPF_REG_AX as the address of arena access. With other JITs the flag is rejected with -EOPNOTSUPP. x86 and arm64 do: - x86: the fault handler finds the register that holds the address through reg2pt_regs[]. Add BPF_REG_AX, r10 of x86, there. - arm64: nothing else is needed. The JIT takes any register as the address and the fault handler reads it by its number. Compile tested only. Any number is an address of arena, NULL and small numbers included, like it is for PTR_TO_ARENA: arena code in C dereferences NULL and relies on the fault being handled. Still rejected: - insn that sees a number on one path and a pointer that is not PTR_TO_ARENA on another. - numbers passed to helpers and kfuncs, except __arena arguments. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-4-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/arm64/net/bpf_jit_comp.c | 5 +++ arch/x86/net/bpf_jit_comp.c | 6 ++++ include/linux/bpf_verifier.h | 2 ++ include/linux/filter.h | 1 + include/uapi/linux/bpf.h | 6 ++++ kernel/bpf/core.c | 6 ++++ kernel/bpf/fixups.c | 39 ++++++++++++++++++++++ kernel/bpf/syscall.c | 4 +++ kernel/bpf/verifier.c | 61 +++++++++++++++++++++++++++------- tools/include/uapi/linux/bpf.h | 6 ++++ 10 files changed, 124 insertions(+), 12 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 17134138fb5c..08725cde0c5a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -3420,6 +3420,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e38466ad44b2..083fcd6cf15b 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -236,6 +236,7 @@ static const int reg2pt_regs[] = { [BPF_REG_7] = offsetof(struct pt_regs, r13), [BPF_REG_8] = offsetof(struct pt_regs, r14), [BPF_REG_9] = offsetof(struct pt_regs, r15), + [BPF_REG_AX] = offsetof(struct pt_regs, r10), }; /* @@ -4673,6 +4674,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 79c4be5c33f4..c51083c761cf 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -672,6 +672,7 @@ struct bpf_insn_aux_data { bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ + bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ u8 alu_state; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. @@ -943,6 +944,7 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ + bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. diff --git a/include/linux/filter.h b/include/linux/filter.h index e42eccb0990e..9339c6131f8f 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1250,6 +1250,7 @@ bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); +bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_large_stack(void); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 4687c3310996..e0ed44b1bbcb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1344,6 +1344,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a67c895ce734..92a87c73aa44 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3435,6 +3435,12 @@ bool __weak bpf_jit_supports_arena(void) return false; } +/* Whether JIT takes BPF_REG_AX as the address of arena access */ +bool __weak bpf_jit_supports_arena_scalar(void) +{ + return false; +} + bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index cb7219ff68bd..206cc9a614a9 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -44,6 +44,31 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } +/* + * The insn accesses arena through a number. JITs add the base of arena + * to the register as it is, so the access goes through the low 32 bits of + * the number in BPF_REG_AX. Registers of the program are not changed: the + * number may be compared or stored later, and on another path the register + * may be PTR_TO_ARENA. + * + * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate + * of st through a number is not blinded. + */ +static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) +{ + bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); + struct bpf_insn *patch = buf; + + *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); + *patch = *insn; + if (load) + patch->src_reg = BPF_REG_AX; + else + patch->dst_reg = BPF_REG_AX; + patch++; + return patch - buf; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1736,6 +1761,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + if (env->insn_aux_data[i + delta].arena_scalar) { + cnt = arena_scalar_access(insn, insn_buf); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + prog = new_prog; + env->prog = prog; + insn = prog->insnsi + i + delta; + goto next_insn; + } + /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4e5377447e1b..654f896af265 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2972,9 +2972,13 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | + BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; + if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) + return -EOPNOTSUPP; + bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 866c0540c68a..fd3c0206bd67 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5260,6 +5260,19 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +/* + * There are no address spaces in Rust, addresses of arena are plain numbers. + * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or + * a store through a number is an access to arena at the low 32 bits of it, + * like the access through PTR_TO_ARENA is. The register stays a number. + */ +static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; +} + static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5290,7 +5303,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno)) + if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -7171,6 +7184,22 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); +/* + * Returns the register to check the access of the current insn with. + * For a number in a program with an arena that is 'arena'. + */ +static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, + struct bpf_reg_state *arena) +{ + if (!is_arena_scalar(env, regno)) + return cur_regs(env) + regno; + + memset(arena, 0, sizeof(*arena)); + arena->type = PTR_TO_ARENA; + env->insn_aux_data[env->insn_idx].arena_scalar = true; + return arena; +} + static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -7178,6 +7207,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -7199,15 +7229,16 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg_type = regs[insn->src_reg].type; + src_reg = mem_access_reg(env, insn->src_reg, &arena); + src_reg_type = src_reg->type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), + insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); @@ -7224,6 +7255,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -7245,11 +7277,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg_type = regs[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -7259,7 +7292,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state *dst_reg; + struct bpf_reg_state arena, *dst_reg; int load_reg; int err; @@ -7304,6 +7337,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } + /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -7312,8 +7348,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } - dst_reg = cur_regs(env) + insn->dst_reg; - /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -19369,15 +19403,17 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg_type = cur_regs(env)[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -22617,6 +22653,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; + env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 4687c3310996..e0ed44b1bbcb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1344,6 +1344,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ From b233f937b98f8e5c88e2978ef36321f888076274 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:03 +0000 Subject: [PATCH 373/660] selftests/bpf: Add tests for arena access through numbers Add tests for load and store through the address that bpf_arena_alloc_pages() returned, without cast_kern: - all sizes, sign extending load, atomics, load-acquire, store-release. - no register is changed by the access, whatever the upper half of the address is, and 64-bit math on the address stays 64-bit. - a number that is not an address of arena, address of the stack included, reads zeroes and writes nowhere at both ends of the range of insn offset. - insn that sees a number on one path and PTR_TO_ARENA on another, followed by access through PTR_TO_ARENA. - store in a callback of bpf_loop(). - NULL that a helper returned and a number that is less than a page are addresses of arena too. - xchg that loads into the register that holds the address. - rejected: prog without arena, prog without BPF_F_ARENA_SCALAR, pointer to stack at the same insn, xchg that loads into the register that holds a pointer to stack, numbers passed to helpers. The tests are for x86 and arm64, other JITs don't take the flag. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-5-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/prog_tests/arena_scalar_blinded.c | 21 + .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_arena_scalar.c | 912 ++++++++++++++++++ tools/testing/selftests/bpf/test_loader.c | 2 + 4 files changed, 937 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c create mode 100644 tools/testing/selftests/bpf/progs/verifier_arena_scalar.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c new file mode 100644 index 000000000000..2ac2e9a652fe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "sysctl_helpers.h" +#include "verifier_arena_scalar.skel.h" + +/* The same tests with constants of the programs blinded */ +void serial_test_arena_scalar_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + RUN_TESTS(verifier_arena_scalar); + sysctl_set_or_fail(harden, NULL, old); +} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8a6d341b754a..460ad10ddc02 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -12,6 +12,7 @@ #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" +#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -198,6 +199,7 @@ void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } +void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c new file mode 100644 index 000000000000..bebc37f501b7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -0,0 +1,912 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, + __u64 flags) __ksym; + +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START (1ull << 32) +#else +#define ARENA_VM_START (1ull << 44) +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); + __ulong(map_extra, ARENA_VM_START); +} arena SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} hash SEC(".maps"); + +/* JITs that take BPF_F_ARENA_SCALAR */ +#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 + +/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ +void __kfunc_btf_root(void) +{ + bpf_arena_alloc_pages(0, 0, 0, 0, 0); +} + +/* Tests start with r6 = address of a new page as the user space sees it, a number */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load and store of every size") +__success __retval(0) +__load_if_JITed() +__naked void ld_st_sizes(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1122334455667788 ll; \ + *(u64 *)(r6 + 0) = r1; \ + *(u32 *)(r6 + 8) = r1; \ + *(u16 *)(r6 + 12) = r1; \ + *(u8 *)(r6 + 14) = r1; \ + *(u64 *)(r6 + 16) = 0x1234; \ + *(u32 *)(r6 + 24) = 0x5678; \ + *(u16 *)(r6 + 28) = 0x9a; \ + *(u8 *)(r6 + 30) = 0xbc; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != r1 goto 9f; \ + r0 = 2; \ + r2 = *(u32 *)(r6 + 8); \ + if r2 != 0x55667788 goto 9f; \ + r0 = 3; \ + r2 = *(u16 *)(r6 + 12); \ + if r2 != 0x7788 goto 9f; \ + r0 = 4; \ + r2 = *(u8 *)(r6 + 14); \ + if r2 != 0x88 goto 9f; \ + r0 = 5; \ + r2 = *(u64 *)(r6 + 16); \ + if r2 != 0x1234 goto 9f; \ + r0 = 6; \ + r2 = *(u32 *)(r6 + 24); \ + if r2 != 0x5678 goto 9f; \ + r0 = 7; \ + r2 = *(u16 *)(r6 + 28); \ + if r2 != 0x9a goto 9f; \ + r0 = 8; \ + r2 = *(u8 *)(r6 + 30); \ + if r2 != 0xbc goto 9f; \ + /* the address is what it was */ \ + r0 = 9; \ + if r6 != r7 goto 9f; \ + r0 = 10; \ + r7 >>= 32; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void ld_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 77; \ + *(u64 *)(r6 + 0) = r1; \ + r6 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r6 != 77 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: sign extending load") +__success __retval(0) +__load_if_JITed() +__naked void ldsx(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 0) = 0x80; \ + .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ + r0 = 1; \ + if r2 != -128 goto 9f; \ + r0 = 2; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, + BPF_REG_2, BPF_REG_6, 0, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: address loaded from arena") +__success __retval(0) +__load_if_JITed() +__naked void ptr_chase(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* page[0] = &page[64]; page[64] = 5; */ \ + r1 = r6; \ + r1 += 64; \ + *(u64 *)(r6 + 0) = r1; \ + *(u64 *)(r1 + 0) = 5; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r2 != r1 goto 9f; \ + r3 = *(u64 *)(r2 + 0); \ + r0 = 2; \ + if r3 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics") +__success __retval(0) +__load_if_JITed() +__naked void atomics(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 8) = 1; \ + r1 = 2; \ + lock *(u64 *)(r6 + 8) += r1; \ + r1 = 4; \ + .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r1 = 8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ + r0 = 2; \ + if r1 != 7 goto 9f; \ + r0 = 8; \ + r1 = 16; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != 8 goto 9f; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != 16 goto 9f; \ + r0 = 5; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: cmpxchg through r0") +__success __retval(0) +__load_if_JITed() +__naked void cmpxchg_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* The address is in r0. The value at the address is not equal to it. */ \ + *(u64 *)(r6 + 0) = 3; \ + r0 = r6; \ + r1 = 5; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void xchg_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 0) = 3; \ + r1 = r6; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != r6 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds a pointer to stack") +__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") +__naked void xchg_into_stack_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r1 = r10; \ + r1 += -8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load-acquire and store-release") +__success __retval(0) +__load_if_JITed() +__naked void load_acq_store_rel(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1234; \ + .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ + .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ + r0 = 1; \ + if r2 != 0x1234 goto 9f; \ + .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ + r0 = 2; \ + if r2 != 0x34 goto 9f; \ + r0 = 3; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), + __imm_insn(load_acquire8_insn, + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void not_in_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + /* nothing is allocated: all loads read 0, stores are dropped */ \ + r6 = 0xdeadbeef00000000 ll; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != 0 goto 9f; \ + r0 = 2; \ + r2 = *(u8 *)(r6 - 32768); \ + if r2 != 0 goto 9f; \ + r6 = 0x12345678ffffffff ll; \ + r0 = 3; \ + r2 = *(u64 *)(r6 + 32760); \ + if r2 != 0 goto 9f; \ + *(u64 *)(r6 + 32760) = 1; \ + *(u8 *)(r6 + 32767) = r2; \ + r1 = 1; \ + lock *(u64 *)(r6 + 32760) += r1; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through the address of the stack as a number") +__success __retval(0) +__load_if_JITed() +__naked void stack_addr_as_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + *(u64 *)(r10 - 8) = 5; \ + r6 = r10; \ + r6 |= 0; \ + /* r6 is a number now. The store goes to arena, not to the stack. */ \ + *(u64 *)(r6 - 8) = 7; \ + r1 = 9; \ + *(u64 *)(r6 - 8) = r1; \ + lock *(u64 *)(r6 - 8) += r1; \ + r2 = *(u64 *)(r10 - 8); \ + r0 = 1; \ + if r2 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: 64-bit math on the address stays 64-bit") +__success __retval(0) +__load_if_JITed() +__naked void alu64_after_access(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r7 = r6; \ + r7 += 8; \ + r7 -= r6; \ + r0 = 1; \ + if r7 != 8 goto 9f; \ + r7 = r6; \ + r7 >>= 32; \ + r0 = 2; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: st of an immediate changes no register") +__success __retval(0) +__load_if_JITed() +__naked void st_keeps_regs(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r0 = r6; \ + r1 = 0x1111; \ + r2 = 0x2222; \ + *(u64 *)(r0 + 0) = 5; \ + r3 = r0; \ + r0 = 1; \ + if r3 != r6 goto 9f; \ + r0 = 2; \ + if r1 != 0x1111 goto 9f; \ + r0 = 3; \ + if r2 != 0x2222 goto 9f; \ + r0 = 0x3333; \ + r1 = r6; \ + *(u32 *)(r1 + 8) = -7; \ + r3 = r0; \ + r0 = 4; \ + if r3 != 0x3333 goto 9f; \ + r0 = 5; \ + if r1 != r6 goto 9f; \ + r0 = 6; \ + r3 = *(u64 *)(r6 + 0); \ + if r3 != 5 goto 9f; \ + r0 = 7; \ + r3 = *(u64 *)(r6 + 8); \ + r4 = 0xfffffff9 ll; \ + if r3 != r4 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number with garbage in the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_garbage(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number without the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_small(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0x1000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics through a number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void atomic_value(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + lock *(u64 *)(r8 + 0) += r1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + .8byte %[fetch_add_insn]; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + .8byte %[xchg_insn]; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ + .8byte %[cmpxchg_insn]; \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to arena at the same insn") +__success __retval(0) +__load_if_JITed() +__naked void mixed_number_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x1234; \ + /* a number that the verifier does not know, 0 at run time */ \ + r7 = *(u64 *)(r6 + 16); \ + r8 = r6; \ + if r7 != 0 goto 1f; \ + .8byte %[cast_kern_insn]; \ +1: *(u8 *)(r8 + 0) = 1; \ + *(u8 *)(r8 + 1) = r7; \ + r2 = *(u8 *)(r8 + 1); \ + lock *(u64 *)(r8 + 24) += r7; \ + r0 = 0; \ + if r7 != 0 goto 9f; \ + /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ + r0 = 2; \ + r2 = *(u64 *)(r8 + 8); \ + if r2 != 0x1234 goto 9f; \ + r0 = 3; \ + r2 = *(u8 *)(r6 + 0); \ + if r2 != 1 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, + BPF_REG_8, BPF_REG_8, 1, 1)) + : __clobber_all); +} + +SEC("syscall") +__description("arena_scalar: no flag, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_flag(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + /* nothing is allocated: the load reads 0 */ \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL with an offset is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr_off(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + r1 += 8; \ + *(u64 *)(r1 + 0) = 5; \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is less than a page is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void small_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r1 &= 0xfff; \ + r0 = *(u64 *)(r1 + 0); \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number is not a pointer for a helper") +__failure __msg("R2 type=scalar expected=") +__naked void helper_arg(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = %[hash] ll; \ + r2 = r6; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_arena_alloc_pages), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to stack at the same insn") +__failure __msg("same insn cannot be used with different pointers") +__load_if_JITed() +__naked void mixed_number_stack(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r10 - 8) = 0; \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r6 = r10; \ + r6 += -8; \ +1: r0 = *(u64 *)(r6 + 0); \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static int st_cb(__u64 idx, void *ctx) +{ + volatile long *p = *(volatile long **)ctx; + + p[idx] = 7; + return 0; +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through a number in a callback") +__success __retval(0) +__load_if_JITed() +int st_in_callback(void *unused) +{ + volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); + + if (!p) + return 100; + bpf_loop(4, st_cb, &p, 0); + return p[0] + p[3] + p[4] - 14; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 25eeb1c1248b..a89890cd56d8 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -580,6 +580,8 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); + } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { + update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) From bed1986a1932942e9db17fbdb2a19d0c4160a1f5 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:04 +0000 Subject: [PATCH 374/660] bpf: Allow names of Rust types and functions in BTF Names of types and functions in BTF that LLVM makes for a Rust program are not C identifiers: [69] STRUCT 'NonNull' size=16 vlen=1 [147] FUNC 'write_fmt' type_id=146 and the kernel rejects such BTF with "Invalid name". scx_simple and scx_cosmos schedulers written in Rust have them in STRUCT, FWD and FUNC, made of letters, digits and " #&()*,:;<>[]{}", 430 characters at most. Allow any printable character in btf_name_valid_identifier(), like it's done for DATASEC. It checks names of types, functions, members, enumerators, variables and arguments, so all of them can have such characters now. The limit of KSYM_NAME_LEN stays. The name of FUNC is a part of the name of the program in kallsyms, where a space would break the parsers. Replace what is not a character of an identifier with '_' there. Tests in prog_tests/btf.c expect "Invalid name" for names with '!' and '*', which are valid now. Put a character that is not printable there. The type name '?foo' is expected to load. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-6-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 15 ++---- kernel/bpf/core.c | 5 ++ tools/testing/selftests/bpf/prog_tests/btf.c | 52 ++++++++++---------- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 8cc17a1cd25c..d27af5d8e495 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -901,16 +901,6 @@ static bool btf_name_offset_valid(const struct btf *btf, u32 offset) return offset < btf->hdr.str_len; } -static bool __btf_name_char_ok(char c, bool first) -{ - if ((first ? !isalpha(c) : - !isalnum(c)) && - c != '_' && - c != '.') - return false; - return true; -} - const char *btf_str_by_offset(const struct btf *btf, u32 offset) { while (offset < btf->start_str_off) @@ -923,20 +913,21 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset) return NULL; } +/* Names in BTF of Rust are not C identifiers. Allow any printable character */ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) { /* offset must be valid */ const char *src = btf_str_by_offset(btf, offset); const char *src_limit; - if (!__btf_name_char_ok(*src, true)) + if (!isprint(*src)) return false; /* set a limit on identifier length */ src_limit = src + KSYM_NAME_LEN; src++; while (*src && src < src_limit) { - if (!__btf_name_char_ok(*src, false)) + if (!isprint(*src)) return false; src++; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 92a87c73aa44..05c89396119a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -589,6 +590,10 @@ bpf_prog_ksym_set_name(struct bpf_prog *prog) prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); + /* the name of a function of Rust is not an identifier */ + for (; *sym; sym++) + if (!isalnum(*sym) && *sym != '_' && *sym != '.') + *sym = '_'; return; } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index df6ad38d287d..87b554067071 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -1987,14 +1987,14 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "typedef (invalid name, invalid identifier)", + .descr = "typedef (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPEDEF_ENC(NAME_TBD, 1), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!int", - .str_sec_size = sizeof("\0__!int"), + .str_sec = "\0__\7int", + .str_sec_size = sizeof("\0__\7int"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "typedef_check_btf", .key_size = sizeof(int), @@ -2112,15 +2112,15 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "fwd type (invalid name, invalid identifier)", + .descr = "fwd type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FWD, 0, 0), 0), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!skb", - .str_sec_size = sizeof("\0__!skb"), + .str_sec = "\0__\7skb", + .str_sec_size = sizeof("\0__\7skb"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "fwd_type_check_btf", .key_size = sizeof(int), @@ -2175,7 +2175,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct type (invalid name, invalid identifier)", + .descr = "struct type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2183,8 +2183,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2217,7 +2217,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct member (invalid name, invalid identifier)", + .descr = "struct member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2225,8 +2225,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A\0B*", - .str_sec_size = sizeof("\0A\0B*"), + .str_sec = "\0A\0B\7", + .str_sec_size = sizeof("\0A\0B\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2260,7 +2260,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum type (invalid name, invalid identifier)", + .descr = "enum type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2269,8 +2269,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2306,7 +2306,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum member (invalid name, invalid identifier)", + .descr = "enum member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(0, @@ -2315,8 +2315,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!", - .str_sec_size = sizeof("\0A!"), + .str_sec = "\0A\7", + .str_sec_size = sizeof("\0A\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2625,14 +2625,14 @@ static struct btf_raw_test raw_tests[] = { .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ - /* void (*)(int a, unsigned int !!!) */ + /* void (*)(int a, unsigned int \7) */ BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), BTF_END_RAW, }, - .str_sec = "\0a\0!!!", - .str_sec_size = sizeof("\0a\0!!!"), + .str_sec = "\0a\0\7", + .str_sec_size = sizeof("\0a\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_proto_type_check_btf", .key_size = sizeof(int), @@ -2775,12 +2775,12 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), - /* void !!!(int a, unsigned int b) */ + /* void \7(int a, unsigned int b) */ BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ BTF_END_RAW, }, - .str_sec = "\0a\0b\0!!!", - .str_sec_size = sizeof("\0a\0b\0!!!"), + .str_sec = "\0a\0b\0\7", + .str_sec_size = sizeof("\0a\0b\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_type_check_btf", .key_size = sizeof(int), @@ -3585,15 +3585,13 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, }, { - .descr = "type name '?foo' is not ok", + .descr = "type name '?foo' is ok", .raw_types = { /* union ?foo; */ BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_FWD, 1, 0), 0), /* [1] */ BTF_END_RAW, }, BTF_STR_SEC("\0?foo"), - .err_str = "Invalid name", - .btf_load_err = true, }, { From 711d8a75c507cb843b4f73278d1f975bf15b11c8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:05 +0000 Subject: [PATCH 375/660] selftests/bpf: Add tests for names of Rust types and functions in BTF Check that BTF with names of STRUCT, FWD, TYPEDEF and FUNC that Rust makes is loaded and that the name of FUNC is an identifier in kallsyms. The test is serial. Programs are not in kallsyms while bpf_jit_harden is set and there are tests that set it for a while. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-7-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/prog_tests/btf.c | 15 ++++ .../selftests/bpf/prog_tests/btf_rust.c | 69 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/btf_rust.c diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 87b554067071..207341f4bd99 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -3593,6 +3593,21 @@ static struct btf_raw_test raw_tests[] = { }, BTF_STR_SEC("\0?foo"), }, +{ + .descr = "names of Rust types and functions are ok", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), 0, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 1, 0), + BTF_FWD_ENC(NAME_NTH(4), 0), /* [3] */ + BTF_TYPEDEF_ENC(NAME_NTH(5), 2), /* [4] */ + BTF_FUNC_PROTO_ENC(0, 1), /* [5] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(6), 1), + BTF_FUNC_ENC(NAME_NTH(7), 5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0u32\0Option<&str>\0__0\0*const str\0{impl#9}<[u8; 4]>\0self\0fmt"), +}, { .descr = "float test #1, well-formed", diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c new file mode 100644 index 000000000000..ce2b31087b59 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#define FUNC_NAME "write_fmt" +#define KSYM_NAME "write_fmt_scx_simple__BpfStream_" + +/* The name of a function of Rust is a part of the name of the program in kallsyms */ +static void test_func_name(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int int_id, proto_id, prog_fd = -1, i; + struct bpf_func_info func_info = {}; + struct bpf_prog_info info = {}; + unsigned long long addr; + __u32 len = sizeof(info); + char sym[128], *p = sym; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "i32", 4, BTF_INT_SIGNED); + ASSERT_GT(int_id, 0, "int"); + proto_id = btf__add_func_proto(btf, int_id); + ASSERT_GT(proto_id, 0, "proto"); + ASSERT_OK(btf__add_func_param(btf, "ctx", int_id), "param"); + func_info.type_id = btf__add_func(btf, FUNC_NAME, BTF_FUNC_STATIC, proto_id); + ASSERT_GT(func_info.type_id, 0, "func"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.prog_btf_fd = btf__fd(btf); + opts.func_info = &func_info; + opts.func_info_cnt = 1; + opts.func_info_rec_size = sizeof(func_info); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, + ARRAY_SIZE(insns), &opts); + if (!ASSERT_GE(prog_fd, 0, "prog load")) + goto out; + if (!ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), "prog info")) + goto out; + if (!info.jited_prog_len) { + test__skip(); + goto out; + } + + p += sprintf(p, "bpf_prog_"); + for (i = 0; i < BPF_TAG_SIZE; i++) + p += sprintf(p, "%02x", info.tag[i]); + sprintf(p, "_%s", KSYM_NAME); + ASSERT_OK(kallsyms_find(sym, &addr), sym); +out: + if (prog_fd >= 0) + close(prog_fd); + btf__free(btf); +} + +/* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ +void serial_test_btf_rust(void) +{ + if (test__start_subtest("func_name")) + test_func_name(); +} From 8e1b3e0945a6fa51c72339898d84fba9f0464b41 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:06 +0000 Subject: [PATCH 376/660] bpf: Allow arguments without names in static functions in BTF Some static functions in BTF of a Rust program have arguments without names: [71] FUNC_PROTO '(anon)' ret_type_id=0 vlen=1 '(anon)' type_id=72 [81] FUNC 'unwrap_failed' type_id=71 linkage=static and the kernel rejects such BTF with "Invalid arg#1". It's 5 of 25 static functions in scx_cosmos and 6 of 24 in scx_simple. The verifier looks at types of the arguments. The names are printed only, as "(anon)" when there is none. Allow such static FUNC. Global functions are checked as before. The test "func (Some arg has no name)" in prog_tests/btf.c has a static function. Make it global to keep the check. Signed-off-by: Alexei Starovoitov Acked-by: Alan Maguire Link: https://lore.kernel.org/bpf/20261002124714.180012-8-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 4 ++++ tools/testing/selftests/bpf/prog_tests/btf.c | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d27af5d8e495..c9d4b709380c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5752,6 +5752,10 @@ static int btf_func_check(struct btf_verifier_env *env, return -EINVAL; } + /* Rust leaves out names of some arguments of static functions */ + if (btf_func_linkage(t) == BTF_FUNC_STATIC) + return 0; + args = (const struct btf_param *)(proto_type + 1); nr_args = btf_type_vlen(proto_type); for (i = 0; i < nr_args; i++) { diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 207341f4bd99..21fdeeb23405 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -2793,7 +2793,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "func (Some arg has no name)", + .descr = "func (Some arg of global func has no name)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ @@ -2802,7 +2802,8 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(0, 2), /* void func(int a, unsigned int) */ - BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_TYPE_ENC(NAME_TBD, /* [4] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), BTF_END_RAW, }, .str_sec = "\0a\0func", From 3dc3a91ec9c7184eb5782c86083c7638e60675a0 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:07 +0000 Subject: [PATCH 377/660] selftests/bpf: Add test for arguments without names in static functions Check that BTF with a static function that has an argument without a name is loaded. Vararg has no name too. Check that a global function with it is loaded as before. Signed-off-by: Alexei Starovoitov Acked-by: Alan Maguire Link: https://lore.kernel.org/bpf/20261002124714.180012-9-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/prog_tests/btf.c | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 21fdeeb23405..642800b80c74 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -2819,6 +2819,57 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid arg#2", }, +{ + .descr = "func (Some arg of static func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + /* static void func(int a, unsigned int) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func (vararg of global func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* void (*)(int a, ...) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + /* void func(int a, ...) */ + BTF_TYPE_ENC(NAME_TBD, /* [3] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + { .descr = "func (Non zero vlen)", .raw_types = { From 9094a158e842323b4487e32e88f01bd45a16af66 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:08 +0000 Subject: [PATCH 378/660] bpf: Allow a variable in DATASEC that is smaller than its type LLVM splits a static of a Rust program into pieces. Every piece is a VAR with the type of the whole static: [223] STRUCT 'BpfCell>' size=32 vlen=1 [236] VAR '..scx_cosmos9TASK_CTXS.0' type_id=223, linkage=static [248] DATASEC '.bss' size=0 vlen=9 type_id=236 offset=24648 size=1 (VAR '..TASK_CTXS.0') and the kernel rejects such BTF with "Invalid size". Allow it. The size of a variable in DATASEC is used in two places: - btf_find_datasec_var() looks for timers, spin locks, kptrs and other special fields. btf_find_field_one() skips a variable when its size is not the size of its type, so there are no special fields in a piece. - btf_datasec_show() prints variables by their types. It would read past the piece and past the end of the map value when the piece is the last one. Don't print the pieces. "global data test #8" and "#10" in prog_tests/btf.c expect "Invalid size". BTF of both is loaded now. The map of #10 is not created, since its value is larger than the section. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-10-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 31 +++++++++++++++----- tools/testing/selftests/bpf/prog_tests/btf.c | 9 ++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c9d4b709380c..0630675377aa 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5397,7 +5397,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { - u32 var_type_id = vsi->type, type_id, type_size = 0; + u32 var_type_id = vsi->type, type_id; const struct btf_type *var_type = btf_type_by_id(env->btf, var_type_id); if (!var_type || !btf_type_is_var(var_type)) { @@ -5412,16 +5412,16 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, return env_stack_push(env, var_type, var_type_id); } + /* + * The variable can be smaller than its type. It's a piece of + * a variable that the compiler split then, with the type of + * the whole variable. + */ type_id = var_type->type; - if (!btf_type_id_size(btf, &type_id, &type_size)) { + if (!btf_type_id_size(btf, &type_id, NULL)) { btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); return -EINVAL; } - - if (vsi->size < type_size) { - btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); - return -EINVAL; - } } env_stack_pop_resolved(env, 0, 0); @@ -5434,6 +5434,16 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } +/* A piece of a variable is smaller than its type, which is the one of the whole variable */ +static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, + const struct btf_var_secinfo *vsi) +{ + u32 size; + + return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || + vsi->size < size; +} + static void btf_datasec_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, @@ -5441,6 +5451,7 @@ static void btf_datasec_show(const struct btf *btf, { const struct btf_var_secinfo *vsi; const struct btf_type *var; + bool comma = false; u32 i; if (!btf_show_start_type(show, t, type_id, data)) @@ -5450,8 +5461,12 @@ static void btf_datasec_show(const struct btf *btf, __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); - if (i) + /* there is less data than the type takes */ + if (btf_var_is_piece(btf, var, vsi)) + continue; + if (comma) btf_show(show, ","); + comma = true; btf_type_ops(var)->show(btf, var, vsi->type, data + vsi->offset, bits_offset, show); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 642800b80c74..24ab62b2834a 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -424,7 +424,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid type", }, { - .descr = "global data test #8, invalid var size", + .descr = "global data test #8, var is smaller than its type", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -457,8 +457,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", }, { .descr = "global data test #9, invalid var size", @@ -498,7 +496,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid size", }, { - .descr = "global data test #10, invalid var size", + .descr = "global data test #10, section is smaller than map value", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -531,8 +529,7 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", + .map_create_err = true, }, { .descr = "global data test #11, multiple section members", From 124a5a2bc0e28989aa007acf6543c826143cbf65 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:09 +0000 Subject: [PATCH 379/660] bpftool: Skip pieces of variables in DATASEC The kernel accepts a variable in DATASEC that is smaller than its type. It's a piece of a variable that the compiler split, with the type of the whole variable. bpftool prints a variable by its type, so it reads past the piece and, when the piece is the last in the section, past the buffer with the value of the map: $ bpftool map dump pinned /sys/fs/bpf/piece ".bss": [{ "whole": 67305985 },{ "piece.0": { "a": [134678021,1994496712,32592,32,0,833,0,414764080 Only the first number is in the map. Skip the pieces in btf_dumper_datasec(), like btf_datasec_show() in the kernel does, and in show_prog_metadata(). Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-11-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/bpftool/btf_dumper.c | 11 +++++++++++ tools/bpf/bpftool/main.h | 2 ++ tools/bpf/bpftool/prog.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index e4075824343f..3267163119dc 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -524,6 +524,15 @@ static int btf_dumper_var(const struct btf_dumper *d, __u32 type_id, return ret; } +/* + * The compiler splits a variable into pieces. Every piece is a VAR with the + * type of the whole variable, so there is less data than the type takes. + */ +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi) +{ + return btf__resolve_size(btf, vsi->type) > vsi->size; +} + static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, const void *data) { @@ -542,6 +551,8 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, jsonw_name(d->jw, btf__name_by_offset(d->btf, t->name_off)); jsonw_start_array(d->jw); for (i = 0; i < vlen; i++) { + if (btf_var_is_piece(d->btf, &vsi[i])) + continue; ret = btf_dumper_do_type(d, vsi[i].type, 0, data + vsi[i].offset); if (ret) break; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index fa2c877a3cbf..6851346a6030 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -121,6 +121,7 @@ struct obj_refs { }; struct btf; +struct btf_var_secinfo; struct bpf_line_info; int build_pinned_obj_table(struct hashmap *table, @@ -239,6 +240,7 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, const void *data); void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, char *func_only, int size); +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi); void btf_dump_linfo_plain(const struct btf *btf, const struct bpf_line_info *linfo, diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a60414aaa494..eb077236d2ea 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -348,7 +348,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { @@ -377,7 +377,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { From 87be058a3b6d32086ee84b0ce280092d82ef3541 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:10 +0000 Subject: [PATCH 380/660] selftests/bpf: Add tests for a variable that is smaller than its type Check that a variable in DATASEC that is smaller than its type is not printed when the map is read from bpffs and when bpftool dumps it. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-12-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/btf_rust.c | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c index ce2b31087b59..daf777cadfda 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_rust.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c @@ -2,6 +2,7 @@ #include #include +#include "bpftool_helpers.h" #define FUNC_NAME "write_fmt" #define KSYM_NAME "write_fmt_scx_simple__BpfStream_" @@ -61,9 +62,81 @@ out: btf__free(btf); } +#define PIN_PATH "/sys/fs/bpf/btf_rust_piece" + +/* + * A piece of a static that LLVM split has the type of the whole static. + * It's the last variable in the section, so its type ends past the map value. + */ +static void test_piece(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int int_id, struct_id, var_id, piece_id, sec_id, map_fd = -1, key = 0; + __u32 value[2] = { 0x11111111, 0x22222222 }; + char line[256] = {}, out[1024] = {}; + struct btf *btf; + FILE *f = NULL; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "u32", 4, 0); + ASSERT_GT(int_id, 0, "int"); + struct_id = btf__add_struct(btf, "Whole", 8); + ASSERT_GT(struct_id, 0, "struct"); + ASSERT_OK(btf__add_field(btf, "a", int_id, 0, 0), "field"); + ASSERT_OK(btf__add_field(btf, "b", int_id, 32, 0), "field"); + var_id = btf__add_var(btf, "CNT", BTF_VAR_STATIC, int_id); + ASSERT_GT(var_id, 0, "var"); + piece_id = btf__add_var(btf, "WHOLE.1", BTF_VAR_STATIC, struct_id); + ASSERT_GT(piece_id, 0, "piece"); + sec_id = btf__add_datasec(btf, ".bss", sizeof(value)); + ASSERT_GT(sec_id, 0, "datasec"); + ASSERT_OK(btf__add_datasec_var_info(btf, var_id, 0, 4), "var info"); + ASSERT_OK(btf__add_datasec_var_info(btf, piece_id, 4, 4), "piece info"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_value_type_id = sec_id; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, ".bss", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "map create")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, value, 0), "map update")) + goto out; + + /* the variable is printed, the piece is not */ + unlink(PIN_PATH); + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin")) + goto out; + f = fopen(PIN_PATH, "r"); + if (!ASSERT_OK_PTR(f, "open")) + goto out; + while (fgets(line, sizeof(line), f) && line[0] == '#') + ; + ASSERT_HAS_SUBSTR(line, "286331153", "var"); + ASSERT_NULL(strstr(line, "572662306"), "piece"); + + /* the same for bpftool */ + if (!ASSERT_OK(get_bpftool_command_output("map dump pinned " PIN_PATH, out, sizeof(out)), + "bpftool")) + goto out; + ASSERT_HAS_SUBSTR(out, "CNT", "var"); + ASSERT_NULL(strstr(out, "WHOLE.1"), "piece"); +out: + if (f) + fclose(f); + unlink(PIN_PATH); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + /* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ void serial_test_btf_rust(void) { if (test__start_subtest("func_name")) test_func_name(); + if (test__start_subtest("piece")) + test_piece(); } From 8ab58ed42b086de4eb4035b4b6a75d29ac7f5e0f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:11 +0000 Subject: [PATCH 381/660] libbpf: Keep global data in arena when the object has .arena.data All memory of a Rust program is arena, global data included. There are no address spaces in Rust to say so. LLVM also drops bounds checks of indexes into constant tables of core that it proved, so the verifier can't check the access as access to map value. The object asks for it with a section named .arena.data. What is in the section doesn't matter: #[used] #[link_section = ".arena.data"] static DATA_IN_ARENA: u8 = 0; or in C: char data_in_arena SEC(".arena.data"); There is nothing to tell to libbpf, so bpftool, veristat and other loaders work with such objects as they are. When the object has the section: - .data, .bss and .rodata sections are appended to arena data after __arena variables, at the alignment of the section. Like __arena variables the data is at the end of arena. - relocations of insns against the sections become relocations against the arena map. - array maps of the sections are not created. - when the object has no arena map libbpf creates one that is as large as the data. bpf_object__find_map_by_name(obj, "arena") finds it and bpf_map__set_max_entries() makes room for allocations. __arena variables still need the arena map to be declared. Read-only sections that stay frozen array maps: - .data.rel.ro and sections that have relocations in them. The kernel recognizes pointers to functions in them by value for callx. - .rodata.str*. They hold const strings for __str arguments of kfuncs. A copy of them is in arena too. The program dereferences pointers that it loads from data, so pointers to data that are stored in data become addresses of arena, wherever the pointer is. The arena map is created ahead of the other maps then, since its address goes into their data. A pointer to .rodata.str* points to the copy. A pointer to a section that is not in arena fails the load: sec '.data': pointer to 'tbl' at offset 8 can't be resolved: sec '.data.rel.ro' is not in arena So does an arena that is pinned or reused. libbpf doesn't mmap it, so its address is not known. Programs of the object are loaded with BPF_F_ARENA_SCALAR, since they access the data through plain numbers. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-13-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/lib/bpf/libbpf.c | 433 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 413 insertions(+), 20 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fdd69aac39bd..cb09ded90773 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -552,6 +552,26 @@ struct bpf_struct_ops { #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" +/* + * An object with this section keeps its global data in arena instead of + * array maps. What is in the section doesn't matter. .data, .bss and .rodata + * sections become a part of the arena map of the object, like __arena global + * variables are. If the object doesn't declare an arena map libbpf creates + * one that is just large enough for the data. bpf_object__find_map_by_name() + * finds it by the name "arena" and bpf_map__set_max_entries() changes its + * size before the object is loaded. + * Read-only sections with pointers to functions and sections with constant + * strings stay frozen array maps. A constant that a helper or a kfunc takes + * by pointer has to be in such section. + * Pointers to data that are stored in data become addresses of arena. + * The load fails if such pointer points to a section that is not in arena, + * or if the arena is pinned or reused. + * + * It's for programs written in Rust. There are no address spaces in Rust, + * the program accesses arena through plain numbers, so programs of the object + * are loaded with BPF_F_ARENA_SCALAR. + */ +#define ARENA_DATA_SEC ".arena.data" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -607,6 +627,13 @@ struct bpf_map { /* pointers to functions in the data of an internal map, see obj->func_ptrs */ struct func_ptr *func_ptrs; size_t func_ptr_cnt; + /* + * Data of the internal map is a part of arena data at arena_off. + * The map is not created, unless it's a copy that is in arena. + */ + bool in_arena; + bool arena_copy; + size_t arena_off; }; enum extern_type { @@ -775,6 +802,17 @@ struct bpf_object { void *arena_data; size_t arena_data_sz; size_t arena_data_off; + bool data_in_arena; + bool arena_mapped; + /* pointers to data in the data of internal maps, when data is in arena */ + struct data_ptr { + int sec_idx; + size_t sec_off; + int targ_sec_idx; + size_t targ_off; + char *sym_name; + } *data_ptrs; + size_t data_ptr_cnt; void *jumptables_data; size_t jumptables_data_sz; @@ -1600,6 +1638,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; + obj->efile.arena_data_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -3081,6 +3120,136 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, return 0; } +static bool map_is_const_str(const struct bpf_map *map) +{ + return map->libbpf_type == LIBBPF_MAP_RODATA && + str_has_pfx(map->real_name, RODATA_SEC ".str"); +} + +static bool map_data_goes_to_arena(const struct bpf_object *obj, const struct bpf_map *map) +{ + int i; + + switch (map->libbpf_type) { + case LIBBPF_MAP_DATA: + case LIBBPF_MAP_BSS: + return true; + case LIBBPF_MAP_RODATA: + break; + default: + return false; + } + + /* + * Const strings that kfuncs and helpers take have to be in a frozen + * map. Pointers to them that are stored in data point to a copy of + * the strings that is in arena. + */ + if (map_is_const_str(map)) + return true; + /* + * Read-only data with pointers stays a frozen map: the kernel + * recognizes pointers to functions in it. + */ + if (str_has_pfx(map->real_name, DATA_REL_RO_SEC)) + return false; + for (i = 0; i < obj->efile.sec_cnt; i++) { + const struct elf_sec_desc *sec = &obj->efile.secs[i]; + + if (sec->sec_type == SEC_RELO && sec->shdr->sh_info == map->sec_idx) + return false; + } + return true; +} + +/* Returns the offset of the data of the map in arena data that is sz bytes so far */ +static size_t map_arena_off(const struct bpf_object *obj, const struct bpf_map *map, size_t sz) +{ + size_t align = obj->efile.secs[map->sec_idx].shdr->sh_addralign; + + return roundup(sz, max(align, sizeof(__u64))); +} + +/* + * Make .data, .bss and .rodata a part of arena data. They follow __arena + * variables of the object, if there are any. + */ +static int bpf_object__init_arena_data(struct bpf_object *obj) +{ + const size_t page_sz = sysconf(_SC_PAGE_SIZE); + size_t sz = obj->arena_data_sz; + struct bpf_map *map; + void *data; + int i; + + if (!obj->data_in_arena) + return 0; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (map_data_goes_to_arena(obj, map)) + sz = map_arena_off(obj, map, sz) + map->def.value_size; + } + if (sz == obj->arena_data_sz) + return 0; + + if (obj->arena_map_idx < 0) { + /* bpf_object__init_user_btf_maps() didn't look: there is no .maps section */ + if (obj->efile.arena_data) { + pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", + ARENA_SEC); + return -ENOENT; + } + + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); + + map->real_name = strdup("arena"); + map->name = strdup("arena"); + if (!map->real_name || !map->name) { + zfree(&map->real_name); + zfree(&map->name); + return -ENOMEM; + } + map->sec_idx = -1; + map->def.type = BPF_MAP_TYPE_ARENA; + map->def.max_entries = roundup(sz, page_sz) / page_sz; + map->def.map_flags = BPF_F_MMAPABLE; + obj->arena_map_idx = map - obj->maps; + } + + data = realloc(obj->arena_data, sz); + if (!data) + return -ENOMEM; + memset(data + obj->arena_data_sz, 0, sz - obj->arena_data_sz); + sz = obj->arena_data_sz; + obj->arena_data = data; + obj->maps[obj->arena_map_idx].mmaped = data; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (!map_data_goes_to_arena(obj, map)) + continue; + sz = map_arena_off(obj, map, sz); + map->arena_off = sz; + memcpy(data + sz, map->mmaped, map->def.value_size); + sz += map->def.value_size; + pr_debug("map '%s': data is in arena at offset %zu\n", map->name, map->arena_off); + if (map_is_const_str(map)) { + map->arena_copy = true; + continue; + } + munmap(map->mmaped, bpf_map_mmap_sz(map)); + /* make bpf_map__initial_value() and bpf_map__set_initial_value() work */ + map->mmaped = data + map->arena_off; + map->autocreate = false; + map->in_arena = true; + } + obj->arena_data_sz = sz; + return 0; +} + static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, const char *pin_root_path) { @@ -3173,6 +3342,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); err = err ?: bpf_object_init_struct_ops(obj); + err = err ?: bpf_object__init_arena_data(obj); return err; } @@ -3987,6 +4157,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!sh) return -LIBBPF_ERRNO__FORMAT; + /* the second pass has to know it when it sees relocations of data */ + name = elf_sec_str(obj, sh->sh_name); + if (name && strcmp(name, ARENA_DATA_SEC) == 0) + obj->data_in_arena = true; + if (sh->sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warn("elf: multiple symbol tables in %s\n", obj->path); @@ -4103,6 +4278,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; + } else if (strcmp(name, ARENA_DATA_SEC) == 0) { + /* the marker, see the first pass */ } else if (strcmp(name, JUMPTABLES_SEC) == 0) { obj->jumptables_data = malloc(data->d_size); if (!obj->jumptables_data) @@ -4127,6 +4304,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) * have pointers to functions. */ if (!section_have_execinstr(obj, targ_sec_idx) && + !(obj->data_in_arena && + (!strcmp(name, ".rel" DATA_SEC) || + str_has_pfx(name, ".rel" DATA_SEC "."))) && strcmp(name, ".rel" RODATA_SEC) && !str_has_pfx(name, ".rel" RODATA_SEC ".") && strcmp(name, ".rel" DATA_REL_RO_SEC) && @@ -4878,6 +5058,10 @@ static int bpf_program__record_reloc(struct bpf_program *prog, reloc_desc->insn_idx = insn_idx; reloc_desc->map_idx = map_idx; reloc_desc->sym_off = sym->st_value; + if (map->in_arena) { + reloc_desc->map_idx = obj->arena_map_idx; + reloc_desc->sym_off += map->arena_off; + } return 0; } @@ -5096,6 +5280,9 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + map->autocreate = autocreate; return 0; } @@ -5376,6 +5563,45 @@ bpf_object__reuse_map(struct bpf_map *map) return 0; } +static struct bpf_map *bpf_object__sec_map(struct bpf_object *obj, int sec_idx) +{ + int i; + + for (i = 0; i < obj->nr_maps; i++) + if (bpf_map__is_internal(&obj->maps[i]) && obj->maps[i].sec_idx == sec_idx) + return &obj->maps[i]; + return NULL; +} + +static int bpf_object__mmap_arena(struct bpf_object *obj, struct bpf_map *map) +{ + int i, err; + + map->mmaped = mmap((void *)(long)map->map_extra, + bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, + map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, + map->fd, 0); + if (map->mmaped == MAP_FAILED) { + err = -errno; + map->mmaped = NULL; + pr_warn("map '%s': failed to mmap arena: %s\n", + map->name, errstr(err)); + return err; + } + if (obj->arena_data) { + memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, + obj->arena_data_sz); + zfree(&obj->arena_data); + } + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *m = &obj->maps[i]; + + if (m->in_arena) + m->mmaped = map->mmaped + obj->arena_data_off + m->arena_off; + } + return 0; +} + static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { @@ -5736,6 +5962,10 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } + /* see bpf_object__relocate_data_ptrs() */ + if (map->def.type == BPF_MAP_TYPE_ARENA && obj->arena_mapped) + continue; + err = map_set_def_max_entries(map); if (err) goto err_out; @@ -5773,22 +6003,9 @@ retry: if (err < 0) goto err_out; } else if (map->def.type == BPF_MAP_TYPE_ARENA) { - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %s\n", - map->name, errstr(err)); + err = bpf_object__mmap_arena(obj, map); + if (err) return err; - } - if (obj->arena_data) { - memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, - obj->arena_data_sz); - zfree(&obj->arena_data); - } } if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { err = init_map_in_map_slots(obj, map); @@ -7803,6 +8020,81 @@ err_out: return err; } +/* + * Turn pointers to data that are stored in data into addresses of arena. + * The arena is created here, ahead of the other maps: the address has to be + * known before the maps that hold the pointers are created. + */ +static int bpf_object__relocate_data_ptrs(struct bpf_object *obj) +{ + struct bpf_map *arena, *map, *targ; + __u64 addr, val; + size_t i; + int err; + + if (!obj->data_ptr_cnt) + return 0; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + if (p->targ_sec_idx < 0) + continue; + map = bpf_object__sec_map(obj, p->sec_idx); + targ = bpf_object__sec_map(obj, p->targ_sec_idx); + if (map && (map->in_arena || map->autocreate) && + (!targ || (!targ->in_arena && !targ->arena_copy))) { + pr_warn("sec '%s': pointer to '%s' at offset %zu can't be resolved: sec '%s' is not in arena\n", + map->real_name, p->sym_name, p->sec_off, + targ ? targ->real_name : ""); + return -LIBBPF_ERRNO__RELOC; + } + } + + if (obj->gen_loader) { + pr_warn("pointers to data in data are not supported by light skeleton\n"); + return -ENOTSUP; + } + + if (obj->arena_map_idx < 0) + return 0; + + arena = &obj->maps[obj->arena_map_idx]; + if (!arena->autocreate) + return 0; + /* libbpf doesn't mmap an arena that it didn't create, so the address is not known */ + if (arena->reused || arena->pin_path) { + pr_warn("map '%s': pointers to data in data are not supported with pinned or reused arena\n", + arena->name); + return -ENOTSUP; + } + + err = bpf_object__create_map(obj, arena, false); + err = err ?: bpf_object__mmap_arena(obj, arena); + if (err) { + pr_warn("map '%s': failed to create: %s\n", arena->name, errstr(err)); + return err; + } + obj->arena_mapped = true; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + map = bpf_object__sec_map(obj, p->sec_idx); + if (!map || (!map->in_arena && !map->autocreate)) + continue; + + addr = (__u64)(unsigned long)arena->mmaped + obj->arena_data_off + p->targ_off; + if (p->targ_sec_idx >= 0) + addr += bpf_object__sec_map(obj, p->targ_sec_idx)->arena_off; + + memcpy(&val, map->mmaped + p->sec_off, sizeof(val)); + val += addr; + memcpy(map->mmaped + p->sec_off, &val, sizeof(val)); + } + return 0; +} + static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { struct bpf_program *prog; @@ -7825,8 +8117,10 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat size_t mmap_sz = bpf_map_mmap_sz(arena_map); if (data_sz > mmap_sz) { - pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", - arena_map->name, mmap_sz, obj->arena_data_sz); + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global %s of size %zu\n", + arena_map->name, mmap_sz, + obj->data_in_arena ? "data" : "__arena variables", + obj->arena_data_sz); return -E2BIG; } @@ -7835,6 +8129,10 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat obj->arena_data_off = mmap_sz - data_sz; } + err = bpf_object__relocate_data_ptrs(obj); + if (err) + return err; + /* Before relocating calls pre-process relocations and mark * few ld_imm64 instructions that points to subprogs. * Otherwise bpf_object__reloc_code() later would have to consider @@ -8067,12 +8365,85 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } +/* + * A pointer to data that is stored in data. When data is in arena + * the program dereferences what it loads from data, so the pointer has + * to be the address of the target in arena. + */ +static int bpf_object__collect_data_ptr(struct bpf_object *obj, const char *relo_sec_name, + int relo_idx, size_t sec_idx, const Elf64_Rel *rel, + const Elf64_Sym *sym) +{ + Elf_Data *scn_data = obj->efile.secs[sec_idx].data; + const char *sym_name = elf_sym_str(obj, sym->st_name) ?: ""; + struct data_ptr *ptrs; + + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_shndx < obj->efile.sec_cnt) + sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)) ?: ""; + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + sym->st_shndx >= obj->efile.sec_cnt || + (sym->st_shndx != obj->efile.arena_data_shndx && + !bpf_object__shndx_is_data(obj, sym->st_shndx)) || + rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_warn("sec '%s': relo #%d: can't resolve pointer to '%s' at offset %zu when data is in arena\n", + relo_sec_name, relo_idx, sym_name, (size_t)rel->r_offset); + return -LIBBPF_ERRNO__RELOC; + } + + ptrs = libbpf_reallocarray(obj->data_ptrs, obj->data_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->data_ptrs = ptrs; + + ptrs[obj->data_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->data_ptr_cnt].sec_off = rel->r_offset; + /* __arena variables are at the start of arena data */ + ptrs[obj->data_ptr_cnt].targ_sec_idx = + sym->st_shndx == obj->efile.arena_data_shndx ? -1 : sym->st_shndx; + ptrs[obj->data_ptr_cnt].targ_off = sym->st_value; + ptrs[obj->data_ptr_cnt].sym_name = strdup(sym_name); + if (!ptrs[obj->data_ptr_cnt].sym_name) + return -ENOMEM; + obj->data_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to '%s'\n", + relo_sec_name, relo_idx, (size_t)rel->r_offset, sym_name); + return 0; +} + +/* Collect pointers in a data section that went to arena */ +static int bpf_object__collect_data_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + int i, err, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + Elf64_Sym *sym; + Elf64_Rel *rel; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + sym = rel ? elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)) : NULL; + if (!sym) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, shdr->sh_info, rel, sym); + if (err) + return err; + } + return 0; +} + /* * Collect pointers to functions in a read-only data section. They are * R_BPF_64_ABS64 relocations against .text section, where the offset of * a static function in the section is stored in place. Relocations in data * sections were ignored before pointers to functions were supported. Those - * that are something else, e.g. pointers to data, still are. + * that are something else, e.g. pointers to data, still are, unless data + * is in arena. */ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) @@ -8108,6 +8479,15 @@ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || !sym_is_subprog(sym, obj->efile.text_shndx)) { + int err; + + if (obj->data_in_arena) { + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, + sec_idx, rel, sym); + if (err) + return err; + continue; + } pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", relo_sec_name, i); continue; @@ -8183,6 +8563,8 @@ static int bpf_object__collect_relos(struct bpf_object *obj) if (obj->efile.secs[idx].sec_type == SEC_RODATA) err = bpf_object__collect_rodata_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_DATA) + err = bpf_object__collect_data_relos(obj, shdr, data); else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) @@ -8476,6 +8858,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; + /* the program accesses its data in arena through plain numbers */ + if (obj->data_in_arena) + load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; load_attr.token_fd = obj->token_fd; @@ -8547,7 +8932,7 @@ retry_load: for (i = 0; i < obj->nr_maps; i++) { map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA) + if (map->libbpf_type != LIBBPF_MAP_RODATA || map->in_arena) continue; if (bpf_prog_bind_map(ret, map->fd, NULL)) { @@ -10111,7 +10496,7 @@ static void bpf_map__destroy(struct bpf_map *map) zfree(&map->init_slots); map->init_slots_sz = 0; - if (map->mmaped && map->mmaped != map->obj->arena_data) + if (map->mmaped && map->mmaped != map->obj->arena_data && !map->in_arena) munmap(map->mmaped, bpf_map_mmap_sz(map)); map->mmaped = NULL; @@ -10196,6 +10581,9 @@ void bpf_object__close(struct bpf_object *obj) close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs); + for (i = 0; i < obj->data_ptr_cnt; i++) + zfree(&obj->data_ptrs[i].sym_name); + zfree(&obj->data_ptrs); if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) @@ -10556,6 +10944,8 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; + if (obj->data_in_arena) + attr.prog_flags |= BPF_F_ARENA_SCALAR; prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); if (!prog_btf_fd && obj->btf) @@ -11562,6 +11952,9 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + if (map->mmaped) { size_t mmap_old_sz, mmap_new_sz; int err; From 373e1223ff00630dc28b2d0fb6ec5fb5cceaccce Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:12 +0000 Subject: [PATCH 382/660] libbpf: Keep format strings of bpf_printk() in .rodata.str bpf_printk(), BPF_SNPRINTF(), BPF_SEQ_PRINTF() and bpf_stream_printk() copy the format into a static const array, which the compiler puts into .rodata. When global data of the object is in arena .rodata is there too and the helper doesn't take the format: R1 type=scalar expected=fp, pkt, pkt_meta, map_key, map_value, mem, ringbuf_mem, buf, trusted_ptr_, ctx String literals are in .rodata.str1.1, which libbpf keeps in a read-only map. No compiler flag moves a named array there. Put the arrays into .rodata.str with a section attribute. An object that uses the macros gets one more map, .rodata.str. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-14-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/lib/bpf/bpf_helpers.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 9d160b5b9c0e..b37b727ef216 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -248,13 +248,21 @@ enum libbpf_tristate { #define ___bpf_fill(arr, args...) \ ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) +/* + * Format strings are in a section of their own. When global data of the object + * is in arena (the object has .arena.data section) libbpf keeps .rodata.str* + * sections in read-only maps, which is where helpers and kfuncs take strings + * from. + */ +#define ___bpf_fmt_sec __attribute__((section(".rodata.str"))) + /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ #define BPF_SEQ_PRINTF(seq, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -272,7 +280,7 @@ enum libbpf_tristate { */ #define BPF_SNPRINTF(out, out_size, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -287,7 +295,7 @@ enum libbpf_tristate { #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD #else -#define BPF_PRINTK_FMT_MOD static const +#define BPF_PRINTK_FMT_MOD static const ___bpf_fmt_sec #endif #define __bpf_printk(fmt, ...) \ @@ -303,7 +311,7 @@ enum libbpf_tristate { */ #define __bpf_vprintk(fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -317,7 +325,7 @@ enum libbpf_tristate { #define bpf_stream_printk(stream_id, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ From 27e24617a83b53e0f008837f7a72ba944a078149 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:13 +0000 Subject: [PATCH 383/660] selftests/bpf: Add test for global data in arena Load the program that reads and writes .data, .bss and .rodata from the object with .arena.data section. Check that initial values set through the skeleton reach the program and that the values written by the program are seen through the skeleton. Also check: - alignment of a section. - pointers to data and to a const string that are stored in data and in read-only data, next to a pointer to a function that callx uses. - bpf_strncmp() with a string literal, bpf_printk() and BPF_SNPRINTF(). - an object that has an arena map and __arena variables. - the load fails when a pointer in data points to a section that is not in arena or to a variable of the kernel. - the load fails when there are pointers in data and the arena is pinned or reused. - the open fails when there are __arena variables and no arena map. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-15-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile.skel | 3 +- .../selftests/bpf/prog_tests/data_in_arena.c | 191 ++++++++++++++++++ .../selftests/bpf/progs/data_in_arena.c | 107 ++++++++++ .../selftests/bpf/progs/data_in_arena_decl.c | 37 ++++ .../bpf/progs/data_in_arena_extern.c | 20 ++ .../selftests/bpf/progs/data_in_arena_fail.c | 20 ++ .../selftests/bpf/progs/data_in_arena_nomap.c | 20 ++ 7 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/data_in_arena.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_decl.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_extern.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_fail.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_nomap.c diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 2e22bb901bf3..765666a64b89 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -20,7 +20,8 @@ ifneq ($(BPF_CC),) BPF_SRCS := $(notdir $(wildcard progs/*.c)) BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ + data_in_arena_extern.c data_in_arena_nomap.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c new file mode 100644 index 000000000000..86dc6798bbde --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "data_in_arena.skel.h" +#include "data_in_arena_decl.skel.h" +#include "data_in_arena_fail.skel.h" + +static int run_prog(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) + return -1; + return topts.retval; +} + +static void run(struct data_in_arena *skel, int counter) +{ + int i; + + ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); + ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); + ASSERT_EQ(skel->data->counter, counter + 1, "counter"); + ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); + for (i = 0; i < 4; i++) + ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); +} + +static void test_in_arena(void) +{ + struct bpf_map_info info = {}; + struct data_in_arena *skel; + __u32 len = sizeof(info); + struct bpf_map *arena; + size_t sz; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); + ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); + ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); + ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); + ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); + ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); + ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); + + /* initial values are set the usual way */ + skel->data->counter = 100; + + if (!ASSERT_OK(data_in_arena__load(skel), "load")) + goto out; + /* there are no maps behind the sections */ + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), + "rodata map"); + run(skel, 100); + + /* pointers to data next to pointers to functions */ + ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); + ASSERT_EQ(skel->data->counter, 102, "counter"); + + /* alignment of sections and pointers to data in data */ + ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); + ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); + ASSERT_EQ(skel->data->x, 43, "x"); + ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); + ASSERT_EQ(skel->data->px, &skel->data->x, "px"); + + /* format strings of bpf_printk() and BPF_SNPRINTF() */ + ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); + ASSERT_STREQ(skel->bss->out, "43-7", "out"); +out: + data_in_arena__destroy(skel); +} + +/* The object has an arena map and __arena variables */ +static void test_declared_arena(void) +{ + struct data_in_arena_decl *skel; + struct bpf_map *map; + int arenas = 0; + + skel = data_in_arena_decl__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_object__for_each_map(map, skel->obj) + arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; + ASSERT_EQ(arenas, 1, "no second arena"); + skel->data->counter = 6; + if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) + goto out; + ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); + ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); + ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); + ASSERT_EQ(skel->data->counter, 8, "counter"); +out: + data_in_arena_decl__destroy(skel); +} + +/* A pointer in data that can't be made an address of arena fails the load */ +static void test_ptr_to_map(void) +{ + struct data_in_arena_fail *skel; + + skel = data_in_arena_fail__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + ASSERT_ERR(data_in_arena_fail__load(skel), "load"); + data_in_arena_fail__destroy(skel); +} + +/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ +static void test_ptr_to_extern(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* __arena variables and no arena map. There is no skeleton: the open fails. */ +static void test_arena_var_no_map(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ +static void test_not_my_arena(bool pin) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); + struct data_in_arena *skel; + struct bpf_map *arena; + int fd = -1; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + if (pin) { + ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); + } else { + fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); + if (!ASSERT_GE(fd, 0, "map_create")) + goto out; + ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); + } + ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); +out: + if (fd >= 0) + close(fd); + data_in_arena__destroy(skel); +} + +void test_data_in_arena(void) +{ +#if !defined(__x86_64__) && !defined(__aarch64__) + /* other JITs don't take BPF_F_ARENA_SCALAR */ + test__skip(); + return; +#endif + if (test__start_subtest("arena")) + test_in_arena(); + if (test__start_subtest("declared_arena")) + test_declared_arena(); + if (test__start_subtest("ptr_to_map")) + test_ptr_to_map(); + if (test__start_subtest("ptr_to_extern")) + test_ptr_to_extern(); + if (test__start_subtest("arena_var_no_map")) + test_arena_var_no_map(); + if (test__start_subtest("pinned_arena")) + test_not_my_arena(true); + if (test__start_subtest("reused_arena")) + test_not_my_arena(false); +} diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c new file mode 100644 index 000000000000..ce3838327d6f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long pair[2] = { 1, 2 }; +int x = 42; +long sum; +long table[4]; +struct { + long v[8]; +} aligned64 __attribute__((aligned(64))); +const volatile int ro = 7; +const volatile long ro_table[4] = { 10, 20, 30, 40 }; + +/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ +const char hello[] SEC(".rodata.str.hello") = "hello"; + +/* pointers to data that are stored in data */ +int *px = &x; +const char *str = hello; +int *const volatile cpx SEC(".data.rel.ro") = &x; + +SEC("syscall") +int use_data(void *ctx) +{ + int i; + + for (i = 0; i < 4; i++) + table[i] = ro_table[i] + i; + sum = counter + ro + pair[0] + pair[1]; + counter++; + __sync_fetch_and_add(&pair[1], 3); + return sum; +} + +typedef int (*op_fn)(int); + +static __noinline int add1(int v) +{ + return v + 1; +} + +/* + * Pointers to functions and to data in read-only data of a program with callx. + * Volatile, so that the compiler doesn't replace the pointers with what + * they point to. + */ +static const volatile struct { + op_fn fn; + int *data; + const char *name; +} ops SEC(".data.rel.ro") = { add1, &x, hello }; + +SEC("syscall") +int use_ops(void *ctx) +{ + /* a program has the arena when its code refers to it */ + counter++; + return ops.fn(*ops.data) + ops.name[1]; +} + +SEC("syscall") +int use_ptrs(void *ctx) +{ + unsigned long addr = (unsigned long)&aligned64; + char local[4] = "abc"; + + /* hide the address from the compiler, it knows that '& 63' is 0 */ + asm volatile ("" : "+r"(addr)); + if (addr & 63) + return 1; + aligned64.v[7] = 7; + if (*px != 42) + return 2; + *px = 43; + if (x != 43 || *cpx != 43) + return 3; + if (str[0] != 'h' || str[4] != 'o' || str[5]) + return 4; + /* the literal is in a map */ + if (bpf_strncmp(local, sizeof(local), "abc")) + return 5; + return 0; +} + +char out[16]; + +/* format strings are in a map */ +SEC("syscall") +int use_printk(void *ctx) +{ + char buf[sizeof(out)]; + int i, n; + + bpf_printk("counter %d", counter); + n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); + for (i = 0; i < sizeof(out); i++) + out[i] = buf[i]; + return n; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c new file mode 100644 index 000000000000..01bc4fea8f0c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_experimental.h" +#include + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); +} arena SEC(".maps"); + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long sum; +const volatile int ro = 7; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +int __arena avar = 11; +#else +int avar = 11; +#endif + +SEC("syscall") +int use_data(void *ctx) +{ + sum = counter + ro + avar; + counter++; + avar++; + return sum; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c new file mode 100644 index 000000000000..dc1ad5ca3bdd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +extern const int bpf_prog_active __ksym; + +/* the variable of the kernel is not in arena */ +const void *kp = &bpf_prog_active; + +SEC("syscall") +int ptr_to_extern(void *ctx) +{ + return *(int *)kp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c new file mode 100644 index 000000000000..ad8afe9afc96 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int x = 42; +/* the table is read-only data with pointers. It's not in arena. */ +int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; +int *const volatile *pp = tbl; + +SEC("syscall") +int ptr_to_map(void *ctx) +{ + return **pp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c new file mode 100644 index 000000000000..da27a8f10d6f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_arena_common.h" + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +/* needs an arena map that is declared. The one that libbpf creates won't do. */ +int __arena avar = 11; + +SEC("syscall") +int arena_var(void *ctx) +{ + return avar + counter; +} + +char _license[] SEC("license") = "GPL"; From e86cb7315451e166f1392032d392f1736fa6853e Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:14 +0000 Subject: [PATCH 384/660] selftests/bpf: Add test for global data of a program in Rust data_in_arena_rust.rs shows why libbpf keeps .data, .bss and .rodata of a program in Rust in arena. A reference in Rust is an address that can be stored in data. The list in the test has a node in each of the three sections. IN_BSS.next is stored by the program, IN_DATA.next is a relocation in .data against .rodata. sum() follows them and reads all nodes with the same insns: 5: w0 = *(u32 *)(r1 + 0x8) 8: r1 = *(u64 *)(r1 + 0x0) The same program without .arena.data section, when the sections are array maps: libbpf: elf: skipping relo section(9) .rel.data for section(8) .data ... 16: (79) r1 = *(u64 *)(r1 +0) ; frame1: R1=scalar() ... 13: (61) r0 = *(u32 *)(r1 +8) R1 invalid mem access 'scalar' The program is built by upstream rustc for its bpfel-unknown-none target. Nothing is done to the output: rustc emits LLVM bitcode and clang makes the object of it for the cpu version of the test_progs flavor. rustc has no prebuilt core for the target, so core is built from the source that comes with rustc, like the kernel does it. That takes 5 seconds. The subtest is skipped when: - there is no rustc or it is older than 1.87. core is built as edition 2024, which it is since then. - rustc comes without the source of core (rustup component add rust-src). - clang is older than 23 or older than LLVM of rustc. It can't read the bitcode then. There are no kfuncs in the program. rustc doesn't emit debug info for extern functions, so there would be no BTF for them. panic=abort is a stop gap. Nothing in the program panics, so the panic handler is not loaded. The series from Yonghong are about to add support for panic=unwind. Tested with rustc 1.95.0-nightly (LLVM 22.1) and clang 24. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-16-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 12 +++- .../testing/selftests/bpf/Makefile.buildvars | 25 +++++++ tools/testing/selftests/bpf/Makefile.skel | 15 ++++ .../selftests/bpf/prog_tests/data_in_arena.c | 25 +++++++ .../selftests/bpf/progs/data_in_arena_rust.rs | 70 +++++++++++++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_rust.rs diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index afa589a27b15..a22be7efd1fa 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -422,6 +422,16 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif +# #![no_std] looks for compiler_builtins too. Nothing of it is used. +ifneq ($(RUST_CORE),) +$(RUST_CORE): $(RUST_CORE_SRC) + $(call msg,RUSTC,,$@) + $(Q)mkdir -p $(@D) + +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< + +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ + $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - +endif + # Generated test list headers define gen_tests_hdr @@ -457,7 +467,7 @@ RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ prog_tests/tests.h map_tests/tests.h \ - $(RUNNER_OBJS) + $(RUNNER_OBJS) $(RUST_CORE) # Runtime fixtures for each test_progs flavor. RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index d2a0c0031b87..ff3476bc40a4 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -109,6 +109,31 @@ HOST_INCLUDE_DIR := $(INCLUDE_DIR) endif RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +# Programs in Rust are built by upstream rustc. It has no prebuilt core for +# the bpf target, so it has to come with the source of core: +# rustup component add rust-src +# core is built as edition 2024, which it is since rustc 1.87. +# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be +# 23 or newer and not older than LLVM of rustc. +# Otherwise RUST_CORE is empty and the tests are skipped. +RUSTC ?= rustc +RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ + /lib/rustlib/src/rust/library/core/src/lib.rs) +ifneq ($(RUST_CORE_SRC),) +ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ + llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ + [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ + [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ + 2>/dev/null && echo y),y) +RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib +endif +endif +# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. +# panic=abort is a stop gap until panic=unwind is supported. +RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ + --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ + -L $(dir $(RUST_CORE)) + DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ifneq ($(CROSS_COMPILE),) CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 765666a64b89..06e297a17156 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -138,4 +138,19 @@ $(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) $(BPFTOOL) $(GEN_SKEL) | $(RDIR) $(Q)$(cmd_bpf_link_skel) +# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. +ifneq ($(RUST_CORE),) +ifeq ($(BPF_CC),$(CLANG)) +RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) +BPF_OBJS += $(RUST_OBJS) + +$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) + $(call msg,RUSTC,$(BINARY),$@) + +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ + $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ + -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ + $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) +endif +endif + endif # BPF_CC diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c index 86dc6798bbde..cb1023507c01 100644 --- a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c @@ -167,6 +167,29 @@ out: data_in_arena__destroy(skel); } +/* The object is there if rustc and clang can build it, see Makefile.buildvars */ +static void test_rust(void) +{ + const char *file = "./data_in_arena_rust.bpf.o"; + struct bpf_object *obj; + + if (access(file, R_OK)) { + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + if (!ASSERT_OK(bpf_object__load(obj), "load")) + goto out; + /* libbpf goes on without BTF when the kernel doesn't take it */ + ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); + ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), + 100 + 20 + 3, "retval"); +out: + bpf_object__close(obj); +} + void test_data_in_arena(void) { #if !defined(__x86_64__) && !defined(__aarch64__) @@ -188,4 +211,6 @@ void test_data_in_arena(void) test_not_my_arena(true); if (test__start_subtest("reused_arena")) test_not_my_arena(false); + if (test__start_subtest("rust")) + test_rust(); } diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs new file mode 100644 index 000000000000..b69128487a0f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Why .data, .bss and .rodata of a program in Rust are in arena. +// +// A reference in Rust is an address. It doesn't say what it points to and it +// can be stored in data. The list below has a node in each of the sections. +// The nodes are linked by references that are in the data: +// IN_BSS.next is stored by the program, +// IN_DATA.next is a relocation in .data against .rodata. +// sum() loads the references back and reads the three nodes with the same insn. +// +// When the sections are array maps libbpf skips the relocation in .data, and +// what sum() loads from a node is a number that can't be dereferenced: +// R1 invalid mem access 'scalar' +// In arena the address of a node is a number to begin with. + +#![no_std] +#![no_main] + +// Tell libbpf to keep .data, .bss and .rodata in arena. +#[used] +#[link_section = ".arena.data"] +static DATA_IN_ARENA: u8 = 0; + +#[used] +#[link_section = "license"] +static LICENSE: [u8; 4] = *b"GPL\0"; + +// panic=abort is a stop gap until panic=unwind is supported. +// Nothing here panics, so the handler is not a part of the program. +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + loop {} +} + +pub struct Node { + val: u32, + next: Option<&'static Node>, +} + +// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. +#[no_mangle] +static IN_RODATA: Node = Node { val: 3, next: None }; +#[no_mangle] +static mut IN_DATA: Node = Node { val: 20, next: Some(&IN_RODATA) }; +#[no_mangle] +static mut IN_BSS: Node = Node { val: 0, next: None }; + +#[inline(never)] +fn sum(mut node: Option<&Node>) -> u32 { + let mut sum = 0; + // The verifier wants a bound. + for _ in 0..8 { + let Some(n) = node else { break }; + sum += n.val; + node = n.next; + } + sum +} + +#[no_mangle] +#[link_section = "syscall"] +pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { + unsafe { + let head = &mut *&raw mut IN_BSS; + head.val = 100; + head.next = Some(&*&raw const IN_DATA); + sum(Some(head)) + } +} From 1ad83c31abfb589dfe3d20a3e904758616d8662f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 3 Oct 2026 15:04:56 +0000 Subject: [PATCH 385/660] selftests/bpf: Fix data_in_arena test with GCC GCC doesn't support indirect calls in BPF: progs/data_in_arena.c: In function 'use_ops': progs/data_in_arena.c:65:1: error: indirect call in function, which are not supported by eBPF Call add1() directly when the test is built with GCC. use_ops() still loads the pointers to data that are next to the pointer to the function in .data.rel.ro and dereferences them. Once the build is fixed data_in_arena/arena_var_no_map fails with GCC: test_arena_var_no_map:FAIL:open unexpected pointer: 0x559fb35d3ca0 GCC doesn't support address spaces. __arena is a type tag only there, so avar is in .data instead of .addr_space.1 and there is nothing wrong with the object. Use __arena_global. It puts the variable into .addr_space.1 with both compilers. Fixes: 27e24617a83b ("selftests/bpf: Add test for global data in arena") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261003150456.168903-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/data_in_arena.c | 5 +++++ tools/testing/selftests/bpf/progs/data_in_arena_nomap.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c index ce3838327d6f..18f38103c0fc 100644 --- a/tools/testing/selftests/bpf/progs/data_in_arena.c +++ b/tools/testing/selftests/bpf/progs/data_in_arena.c @@ -61,7 +61,12 @@ int use_ops(void *ctx) { /* a program has the arena when its code refers to it */ counter++; +#ifdef __clang__ return ops.fn(*ops.data) + ops.name[1]; +#else + /* gcc doesn't support indirect calls */ + return add1(*ops.data) + ops.name[1]; +#endif } SEC("syscall") diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c index da27a8f10d6f..ccf66321eb36 100644 --- a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c +++ b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c @@ -9,7 +9,7 @@ char data_in_arena SEC(".arena.data"); int counter = 5; /* needs an arena map that is declared. The one that libbpf creates won't do. */ -int __arena avar = 11; +int __arena_global avar = 11; SEC("syscall") int arena_var(void *ctx) From 99dc1ba542420db6b8df209744f55cc52466ad91 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 3 Oct 2026 20:33:26 +0000 Subject: [PATCH 386/660] selftests/bpf: Format data_in_arena_rust.rs with rustfmt 'make rustfmtcheck' checks every .rs file in the tree and fails since the test in Rust was added: Diff in tools/testing/selftests/bpf/progs/data_in_arena_rust.rs:42: #[no_mangle] static IN_RODATA: Node = Node { val: 3, next: None }; #[no_mangle] -static mut IN_DATA: Node = Node { val: 20, next: Some(&IN_RODATA) }; +static mut IN_DATA: Node = Node { + val: 20, + next: Some(&IN_RODATA), +}; Format the file with rustfmt. The code and the data of the program don't change. Fixes: e86cb7315451 ("selftests/bpf: Add test for global data of a program in Rust") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261003203326.1086454-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/data_in_arena_rust.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs index b69128487a0f..3dd9d4208232 100644 --- a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs +++ b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs @@ -42,7 +42,10 @@ pub struct Node { #[no_mangle] static IN_RODATA: Node = Node { val: 3, next: None }; #[no_mangle] -static mut IN_DATA: Node = Node { val: 20, next: Some(&IN_RODATA) }; +static mut IN_DATA: Node = Node { + val: 20, + next: Some(&IN_RODATA), +}; #[no_mangle] static mut IN_BSS: Node = Node { val: 0, next: None }; From b65cc81792eb07436879d9f0be8ebd18a78fffb4 Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Sun, 27 Sep 2026 20:04:21 +0800 Subject: [PATCH 387/660] bpf: Destroy callee-local dynptrs on subprog return A data slice is valid only while the specific dynptr that produced it remains valid. prepare_func_exit() copies a subprogram return value into the caller and then frees the callee state without applying the normal destruction semantics to dynptrs in the callee stack. A callee can therefore derive a slice from a local dynptr, return or spill the slice into its caller, and then let the dynptr disappear with the callee frame. The slice retains the id of a dynptr that is no longer in the verifier state, so the verifier continues to accept accesses through it. Before the object relationship refactor, bpf_dynptr_data() slices from referenced dynptrs also carried the shared reference id, allowing a later release to catch some of these escaped slices. The refactor made those slices precise children of their source dynptr and exposed the missing teardown as an accepted stale access even after the shared resource is released. Add destroy_dynptrs_in_stack_slots() to apply the existing dynptr teardown to an inclusive range of stack slots. Reuse it for dynptr initialization, variable-offset stack writes, and the entire callee stack before freeing the frame. The teardown rejects losing the last dynptr for a referenced resource and invalidates the dynptr and all its descendants, including slices held in the caller. Propagate errors through prepare_func_exit(). Reuse the existing teardown API and diagnostic. Update the affected callback test expectation in this change so the commit remains test-clean. Fixes: 308c7a0ae885 ("bpf: Refactor object relationship tracking and fix dynptr UAF bug") Reported-by: Sashiko Suggested-by: Amery Hung Signed-off-by: Xu Yunxiang Reviewed-by: Amery Hung Acked-by: Ihor Solodrai Link: https://lore.kernel.org/r/CAMB2axOmdHwSrHihSRskDywkCmEGQOX+6dZPN_A9u-HhxY3UDA@mail.gmail.com Link: https://lore.kernel.org/r/CAMB2axMmrTb+UZ84UD48MwMtXbk1s9bWtreU3AOfjzU0fPT0BA@mail.gmail.com Link: https://lore.kernel.org/bpf/20260927120422.1042107-2-xyx2021@mail.ustc.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 40 +++++++++++++------ .../testing/selftests/bpf/progs/dynptr_fail.c | 4 +- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fd3c0206bd67..353bde9ae227 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -723,6 +723,21 @@ static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); +static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env, + struct bpf_func_state *state, + int first_spi, int last_spi) +{ + int spi, err; + + for (spi = first_spi; spi <= last_spi; spi++) { + err = destroy_if_dynptr_stack_slot(env, state, spi); + if (err) + return err; + } + + return 0; +} + static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type, int insn_idx, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) @@ -736,7 +751,7 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return spi; /* We cannot assume both spi and spi - 1 belong to the same dynptr, - * hence we need to call destroy_if_dynptr_stack_slot twice for both, + * hence we need to destroy dynptrs in both slots, * to ensure that for the following example: * [d1][d1][d2][d2] * spi 3 2 1 0 @@ -744,10 +759,7 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ * case they do belong to same dynptr, second call won't see slot_type * as STACK_DYNPTR and will simply skip destruction. */ - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - err = destroy_if_dynptr_stack_slot(env, state, spi - 1); + err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi); if (err) return err; @@ -3851,14 +3863,10 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true; - for (i = min_off; i < max_off; i++) { - int spi; - - spi = bpf_get_spi(i); - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - } + err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1), + bpf_get_spi(min_off)); + if (err) + return err; /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { @@ -11678,6 +11686,12 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } } + /* Invalidate callee-local dynptrs and their slices before the frame goes away. */ + err = destroy_dynptrs_in_stack_slots(env, callee, 0, + callee->allocated_stack / BPF_REG_SIZE - 1); + if (err) + return err; + /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration * converges is_state_visited() would prune that visit eventually. diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 148cf4417322..78a621a25178 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -125,9 +125,9 @@ static int missing_release_callback_fn(__u32 index, void *data) return 0; } -/* Any dynptr initialized within a callback must have bpf_dynptr_put called */ +/* A callback cannot return with the last dynptr for a referenced resource. */ SEC("?raw_tp") -__failure __msg("Unreleased reference id") +__failure __msg("cannot overwrite referenced dynptr") int ringbuf_missing_release_callback(void *ctx) { bpf_loop(10, missing_release_callback_fn, NULL, 0); From b04ba9375a9e73f554183ea851cecce475cd1f80 Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Sun, 27 Sep 2026 20:04:22 +0800 Subject: [PATCH 388/660] selftests/bpf: Test dynptr teardown on subprog return Add failure tests for slices that escape a subprogram from a dynptr in the callee stack. Cover immediate use of a ring buffer clone slice, release-time invalidation of a caller dynptr slice after it crosses a subprogram return, and immediate use of an skb dynptr slice in a qdisc program. Cover the error path where a subprogram would otherwise lose the last dynptr capable of releasing a referenced resource. Also add a success control which derives a slice from a caller-owned dynptr and creates a separate clone in the callee. Destroying that callee-local clone on return must not invalidate the caller dynptr's slice. Signed-off-by: Xu Yunxiang Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260927120422.1042107-3-xyx2021@mail.ustc.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/bpf_qdisc.c | 2 + ...disc_fail__invalid_dynptr_returned_slice.c | 76 +++++++++++++++ .../testing/selftests/bpf/progs/dynptr_fail.c | 95 +++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 6dbd1487343c..122ecb7e98e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -11,6 +11,7 @@ #include "bpf_qdisc_fail__invalid_dynptr.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_returned_slice.skel.h" #include "bpf_qdisc_fail__untrusted_write.skel.h" #include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" @@ -230,6 +231,7 @@ void test_ns_bpf_qdisc(void) test_incompl_ops(); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_returned_slice); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); RUN_TESTS(bpf_qdisc_fail__untrusted_write); RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c new file mode 100644 index 000000000000..8217f4c4c00c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +static __noinline struct ethhdr *slice_in_subprog(struct sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + return bpf_dynptr_slice(&ptr, 0, NULL, sizeof(struct ethhdr)); +} + +SEC("struct_ops") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(invalid_dynptr_returned_slice, struct sk_buff *skb, + struct Qdisc *sch, struct bpf_sk_buff_ptr *to_free) +{ + struct ethhdr *hdr; + + hdr = slice_in_subprog(skb); + if (!hdr) { + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; + } + + /* this should fail */ + proto = hdr->h_proto; + + bpf_qdisc_skb_drop(skb, to_free); + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr_returned_slice, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 78a621a25178..c2247b38c849 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1895,6 +1895,101 @@ int clone_invalidate4(void *ctx) return 0; } +static __noinline void clone_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + bpf_dynptr_clone(ptr, &clone); + *data = bpf_dynptr_data(&clone, 0, sizeof(val)); +} + +static __noinline void caller_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + *data = bpf_dynptr_data(ptr, 0, sizeof(val)); + bpf_dynptr_clone(ptr, &clone); +} + +static __noinline void reserve_dynptr_in_subprog(void) +{ + struct bpf_dynptr ptr; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); +} + +/* A subprogram cannot lose the last dynptr that can release a resource. */ +SEC("?raw_tp") +__failure __msg("cannot overwrite referenced dynptr") +int referenced_dynptr_lost_on_subprog_return(void *ctx) +{ + reserve_dynptr_in_subprog(); + + return 0; +} + +/* + * Destroying a callee-local clone on return must not invalidate a slice whose + * source dynptr belongs to the caller. + */ +SEC("?raw_tp") +__success +int caller_dynptr_slice_across_subprog_valid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + if (data) + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice derived from a callee-local clone is invalid after the subprogram + * returns. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int callee_dynptr_slice_invalid_after_return(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + clone_slice_in_subprog(&ptr, &data); + if (data) + /* this should fail */ + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice from a caller-owned dynptr survives the subprogram return, but + * releasing the shared reservation must invalidate it. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int caller_dynptr_slice_release_after_subprog_invalid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + bpf_ringbuf_submit_dynptr(&ptr, 0); + if (data) + /* this should fail */ + *data = 123; + + return 0; +} + /* Invalidating a dynptr should invalidate any data slices * of its parent */ From 5b6beb95f7b4bde8333c588543a2400e52805cba Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:19 -0700 Subject: [PATCH 389/660] bpf: Represent stack access effects with arg_access_info The following patches in the series add support for may_write property inference. To properly track this property there is a need to distinguish three ways the helpers/kfuncs affect stack: - read; - write and destroy all previous stack tracking information for affected slots; - both read and write the stack slots, potentially preserving some of the tracking information. The later property can't be encoded by signed access_byte convention. Hence, introduce arg_access_info with a size and separate may_read, may_write and must_write bits. Convert the liveness access recording paths to use it. Proper may_write calculation and support will be added in the following patches. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-1-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 8 +++ kernel/bpf/liveness.c | 96 +++++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c51083c761cf..d4f32acba236 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1723,6 +1723,14 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn); void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs); +/* Stack effects used for state pruning; must_write implies may_write. */ +struct arg_access_info { + u32 size; /* Maximum extent; U32_MAX if unknown. */ + u8 may_read:1; /* Incoming contents or initialization may be needed. */ + u8 may_write:1; + u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ +}; + s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index cd9523f69298..7b317fb5ca82 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1437,17 +1437,14 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, } /* - * Record access_bytes from helper/kfunc or load/store insn. - * access_bytes > 0: stack read - * access_bytes < 0: stack write - * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read - * access_bytes == 0: no access - * + * Record reads for every touched half-slot. A definite + * write requires full coverage, a known size, and a single possible offset. */ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, - s64 access_bytes, u32 frame, u32 insn_idx) + struct arg_access_info info, u32 frame, u32 insn_idx) { s32 slot_hi, slot_lo; + int err; if (fp_off >= 0) /* @@ -1456,49 +1453,53 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, * by the main verifier pass later. */ return 0; - if (access_bytes == S64_MIN) { + + if (info.may_read && info.size == U32_MAX) { /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; return mark_stack_read(instance, frame, insn_idx, 0, slot_hi); } - if (access_bytes > 0) { + if (info.may_read) { /* Mark any touched slot as use */ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); - return mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); - } else if (access_bytes < 0) { + slot_lo = max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); + err = mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.must_write && info.size != U32_MAX) { /* Mark only fully covered slots as def */ - access_bytes = -access_bytes; slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; - slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); + slot_lo = max_t(s32, (-fp_off - info.size + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi); } return 0; } -/* - * 'arg' is FP-derived argument to helper/kfunc or load/store that - * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. - */ -static int record_stack_access(struct bpf_verifier_env *env, struct func_instance *instance, +/* Record access through a pointer with a known frame and possibly known offsets. */ +static int record_stack_access(struct bpf_verifier_env *env, + struct func_instance *instance, const struct arg_track *arg, - s64 access_bytes, u32 frame, u32 insn_idx) + struct arg_access_info info, u32 frame, u32 insn_idx) { int i, err; - if (access_bytes == 0) + if (!info.size) return 0; if (arg->off_cnt == 0) { - if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read_all(env, instance, frame, insn_idx); + if (info.may_read) { + err = mark_stack_read_all(env, instance, frame, insn_idx); + if (err) + return err; + } return 0; } - if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) + if (info.size != U32_MAX && info.must_write && arg->off_cnt != 1) /* multi-offset write cannot set stack_def */ return 0; for (i = 0; i < arg->off_cnt; i++) { - err = record_stack_access_off(instance, arg->off[i], access_bytes, frame, insn_idx); + err = record_stack_access_off(instance, arg->off[i], info, frame, insn_idx); if (err) return err; } @@ -1533,9 +1534,10 @@ static int record_load_store_access(struct bpf_verifier_env *env, struct arg_track *at, int insn_idx) { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + struct arg_access_info info = {}; int depth = instance->depth; - s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); u8 class = BPF_CLASS(insn->code); + bool read = false, write = false; struct arg_track resolved, *ptr; int oi; @@ -1552,23 +1554,26 @@ static int record_load_store_access(struct bpf_verifier_env *env, switch (class) { case BPF_LDX: ptr = &at[insn->src_reg]; + read = true; break; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) { if (insn->imm == BPF_STORE_REL) - sz = -sz; + write = true; + else + read = true; if (insn->imm == BPF_LOAD_ACQ) ptr = &at[insn->src_reg]; else ptr = &at[insn->dst_reg]; } else { ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; } break; case BPF_ST: ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; break; default: return 0; @@ -1587,32 +1592,53 @@ static int record_load_store_access(struct bpf_verifier_env *env, ptr = &resolved; } + info.may_read = read; + info.may_write = write; + info.must_write = write; + info.size = bpf_size_to_bytes(BPF_SIZE(insn->code)); if (ptr->frame >= 0 && ptr->frame <= depth) - return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx); + return record_stack_access(env, instance, ptr, info, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) return record_imprecise(env, instance, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } +/* Adapt the signed helper/kfunc access size until both producers are converted. */ +static struct arg_access_info stack_access_info(s64 bytes) +{ + bool write = bytes < 0 && bytes != S64_MIN && -bytes < U32_MAX; + + return (struct arg_access_info) { + .size = bytes == S64_MIN + ? U32_MAX + : min_t(u64, bytes < 0 ? -bytes : bytes, U32_MAX), + .may_read = bytes > 0 || bytes == S64_MIN, + .may_write = write, + .must_write = write, + }; +} + static int record_arg_access(struct bpf_verifier_env *env, struct func_instance *instance, struct bpf_insn *insn, struct arg_track *at, int arg_idx, int insn_idx) { + struct arg_access_info info; int depth = instance->depth; int frame = at->frame; int err = 0; - s64 bytes; if (!arg_is_fp(at)) return 0; if (bpf_helper_call(insn)) { - bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = stack_access_info(bpf_helper_stack_access_bytes(env, insn, + arg_idx, insn_idx)); } else if (bpf_pseudo_kfunc_call(insn)) { - bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = stack_access_info(bpf_kfunc_stack_access_bytes(env, insn, + arg_idx, insn_idx)); } else { for (int f = 0; f <= depth; f++) { err = mark_stack_read_all(env, instance, f, insn_idx); @@ -1621,11 +1647,11 @@ static int record_arg_access(struct bpf_verifier_env *env, } return 0; } - if (bytes == 0) + if (!info.size) return 0; if (frame >= 0 && frame <= depth) - err = record_stack_access(env, instance, at, bytes, frame, insn_idx); + err = record_stack_access(env, instance, at, info, frame, insn_idx); else if (frame == ARG_IMPRECISE) err = record_imprecise(env, instance, at->mask, insn_idx); return err; From 830e4372de5c8e1e0bbea925e74717b8fc259dfd Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:20 -0700 Subject: [PATCH 390/660] bpf: Describe helper stack accesses with arg_access_info Adjust bpf_helper_stack_access_bytes() to return results as a struct arg_access_info. The result is based on bpf_func_proto->argX_type value as in the following table: | Argument flags | allow_uninit_stack | Meaning | |----------------------+--------------------+----------------------| | MEM_WRITE | true | may_read, must_write | | MEM_WRITE+MEM_UNINIT | true | must_write | | MEM_WRITE | false | may_read, must_write | | MEM_WRITE+MEM_UNINIT | false | may_read, may_write | | 0 | true/false | may_read | MEM_WRITE now produces must_write when allow_uninit_stack is true: the verifier discards previous stack knowledge over the affected range. Previously, liveness only treated MEM_UNINIT this way. Also use resolve_map_arg_type() to account for bpf_map_peek_elem() treating its buffer as an input for bloom filters. Annotate ringbuf dynptr submit and discard arguments with MEM_WRITE. Their existing OBJ_RELEASE handling already invalidates the dynptr, but the prototypes need to describe the write as well as the release. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-2-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 6 +-- kernel/bpf/liveness.c | 5 +-- kernel/bpf/ringbuf.c | 4 +- kernel/bpf/verifier.c | 81 +++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d4f32acba236..5c99c024c309 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1731,9 +1731,9 @@ struct arg_access_info { u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ }; -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 7b317fb5ca82..353b52031577 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1604,7 +1604,7 @@ static int record_load_store_access(struct bpf_verifier_env *env, return 0; } -/* Adapt the signed helper/kfunc access size until both producers are converted. */ +/* Adapt the signed kfunc access size until its producer is converted. */ static struct arg_access_info stack_access_info(s64 bytes) { bool write = bytes < 0 && bytes != S64_MIN && -bytes < U32_MAX; @@ -1634,8 +1634,7 @@ static int record_arg_access(struct bpf_verifier_env *env, return 0; if (bpf_helper_call(insn)) { - info = stack_access_info(bpf_helper_stack_access_bytes(env, insn, - arg_idx, insn_idx)); + info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { info = stack_access_info(bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx)); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 3f1013d80544..447863ece26a 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -722,7 +722,7 @@ BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = { .func = bpf_ringbuf_submit_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; @@ -741,7 +741,7 @@ BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { .func = bpf_ringbuf_discard_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 353bde9ae227..bbb81ba0f2dc 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14358,35 +14358,32 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, } /* - * Determine how many bytes a helper accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't read anything directly - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a helper's stack access through argument @arg (0-based, R1-R5). + * must_write means the verifier destroys prior state throughout size bytes. + * In other words, must_write follows the verifier's model of the + * function's behaviour, which is safe to use for is_state_visited() pruning. */ -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; const struct bpf_func_proto *fn; + enum bpf_access_type access_type; enum bpf_arg_type at; - bool full_write; - s64 size; + bool exact_size = true; + u64 size = U32_MAX; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) - return S64_MIN; + return info; at = fn->arg_type[arg]; - /* - * Generic outputs may leave bytes untouched. Keep prior initialization - * live when the caller cannot read uninitialized bytes. Constructors of - * special objects, such as dynptrs, still define their storage. - */ - full_write = (at & MEM_UNINIT) && - (!arg_type_is_raw_mem(at) || env->allow_uninit_stack); + access_type = func_arg_access_type(at); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -14409,6 +14406,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn i = aux->const_reg_vals[map_reg]; if (i < env->used_map_cnt) { + /* Bloom-filter peek reads the value buffer. */ + if (!is_key && insn->imm == BPF_FUNC_map_peek_elem && + env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + access_type = BPF_READ; size = is_key ? env->used_maps[i]->key_size : env->used_maps[i]->value_size; goto out; @@ -14418,7 +14419,12 @@ scan_all_maps: * Map pointer is not known at this call site (e.g. different * maps on merged paths). Conservatively return the largest * key_size or value_size across all maps used by the program. + * This is only an upper bound, so it cannot establish a definite + * write. Map-dependent argument types can also turn an output + * into an input, so conservatively retain a read dependency. */ + exact_size = false; + access_type |= BPF_READ; val = 0; for (i = 0; i < env->used_map_cnt; i++) { struct bpf_map *map = env->used_maps[i]; @@ -14434,7 +14440,7 @@ scan_all_maps: } } if (!val) - return S64_MIN; + return info; size = val; goto out; } @@ -14448,18 +14454,12 @@ scan_all_maps: int size_reg = BPF_REG_1 + arg + 1; if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + size = aux->const_reg_vals[size_reg]; goto out; } - /* - * Size arg is const on each path but differs across merged - * paths. Reads may extend anywhere up to the frame top. - */ - if (full_write) - return 0; - return S64_MIN; } - return S64_MIN; + /* Preserve access directions even when the extent is unknown. */ + goto out; case ARG_PTR_TO_DYNPTR: size = BPF_DYNPTR_SIZE; break; @@ -14469,18 +14469,21 @@ scan_all_maps: * doesn't access stack. The callback subprog does and it's * analyzed separately. */ - return 0; + return (struct arg_access_info) {}; default: - return S64_MIN; + return info; } out: - /* - * Other accesses keep the previous state live, including untouched bytes - * of an unprivileged generic output. - */ - if (full_write) - return -size; - return size; + info.size = min_t(u64, size, U32_MAX); + info.may_read = !!(access_type & BPF_READ); + info.may_write = !!(access_type & BPF_WRITE); + info.must_write = info.may_write && exact_size && info.size != U32_MAX; + /* Generic unprivileged outputs retain prior initialization state. */ + if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) { + info.may_read = true; + info.must_write = false; + } + return info; } /* From 8f3bd4537faad487a1aed4df24c1af4587904568 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:21 -0700 Subject: [PATCH 391/660] bpf: Describe kfunc stack accesses with arg_access_info Return kfunc stack effects through arg_access_info while preserving the existing BTF size lookup and read/def classification. Retain __sz/__szk handling and conservatively mark pointer arguments as possible writes. Remove the temporary signed-size adapter from liveness. Add liveness assertions for privileged and unprivileged __uninit outputs. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-3-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 6 +-- kernel/bpf/liveness.c | 18 +------- kernel/bpf/verifier.c | 43 ++++++++++--------- .../bpf/progs/verifier_kfunc_uninit.c | 6 +++ 4 files changed, 33 insertions(+), 40 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5c99c024c309..eafe79929dfa 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1734,9 +1734,9 @@ struct arg_access_info { struct arg_access_info bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); int bpf_stack_liveness_init(struct bpf_verifier_env *env); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 353b52031577..d4089a488784 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1604,21 +1604,6 @@ static int record_load_store_access(struct bpf_verifier_env *env, return 0; } -/* Adapt the signed kfunc access size until its producer is converted. */ -static struct arg_access_info stack_access_info(s64 bytes) -{ - bool write = bytes < 0 && bytes != S64_MIN && -bytes < U32_MAX; - - return (struct arg_access_info) { - .size = bytes == S64_MIN - ? U32_MAX - : min_t(u64, bytes < 0 ? -bytes : bytes, U32_MAX), - .may_read = bytes > 0 || bytes == S64_MIN, - .may_write = write, - .must_write = write, - }; -} - static int record_arg_access(struct bpf_verifier_env *env, struct func_instance *instance, struct bpf_insn *insn, @@ -1636,8 +1621,7 @@ static int record_arg_access(struct bpf_verifier_env *env, if (bpf_helper_call(insn)) { info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { - info = stack_access_info(bpf_kfunc_stack_access_bytes(env, insn, - arg_idx, insn_idx)); + info = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { err = mark_stack_read_all(env, instance, f, insn_idx); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bbb81ba0f2dc..8987c9a69d10 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14487,28 +14487,29 @@ out: } /* - * Determine how many bytes a kfunc accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't access memory through that argument (ex: not a pointer) - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a kfunc's stack access through argument slot @arg (0-based). + * As for helpers, must_write describes destruction of prior verifier state, + * rather than a guarantee that the callee writes every byte at runtime. */ -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; u32 i, slot, nargs, type_size; - s64 size; + u64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) - return S64_MIN; + return info; btf = meta.btf; args = btf_params(meta.func_proto); @@ -14523,11 +14524,11 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * for (i = 0, slot = 0; i < nargs && slot < arg; i++) slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); if (i >= nargs || slot != arg) - return 0; + return (struct arg_access_info) {}; t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) - return 0; + return (struct arg_access_info) {}; /* dynptr: fixed 16-byte on-stack representation */ if (is_kfunc_arg_dynptr(btf, &args[i])) { @@ -14543,11 +14544,11 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * if (size_reg <= MAX_BPF_FUNC_REG_ARGS && (aux->const_reg_mask & BIT(size_reg))) { - size = (s64)aux->const_reg_vals[size_reg]; + size = aux->const_reg_vals[size_reg]; goto out; } /* Unknown size: the read may extend anywhere up to the frame top. */ - return S64_MIN; + return info; } /* fixed-size pointed-to type: resolve via BTF */ @@ -14557,15 +14558,17 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - return S64_MIN; + return info; out: + info.size = min_t(u64, size, U32_MAX); /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) - return -size; + info.may_read = false; if (is_kfunc_arg_uninit(btf, &args[i]) && (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) - return -size; - return size; + info.may_read = false; + info.must_write = !info.may_read && info.size != U32_MAX; + return info; } /* check special kfuncs and return: diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c index ff2fb36a0260..20ed71a54686 100644 --- a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c @@ -18,6 +18,9 @@ void __kfunc_btf_root(void) SEC("tc") __success __retval(10) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_struct{{.*}}; def: fp0-8 fp0-16") +__msg_unpriv("call bpf_kfunc_test_uninit_struct{{.*}}; use: fp0-8 fp0-16") __flag(BPF_F_TEST_STATE_FREQ) __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __prepare_priv @@ -44,6 +47,9 @@ __naked void struct_poisoned_at_checkpoint(void) SEC("tc") __success __retval(0x2a2a2a2a) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_mem{{.*}}; def: fp0-8") +__msg_unpriv("call bpf_kfunc_test_uninit_mem{{.*}}; use: fp0-8") __flag(BPF_F_TEST_STATE_FREQ) __caps_unpriv(CAP_BPF | CAP_NET_ADMIN) __prepare_priv From ac43e0393f44931bd2d84ab682e78b1f08f9f1ec Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:22 -0700 Subject: [PATCH 392/660] bpf: Track may_write flags in liveness Track stack slots that instructions may modify, in order to account for such writes when constructing SCEV expressions. Consume the may_write flag in arg_access_info separately from reads and definite writes. Include partially covered slots and all candidate offsets; imprecise writes mark whole candidate frames. Possible writes do not kill liveness. Merge analyzed instances as: may_write(dst) |= may_write(src) must_write(dst) &= must_write(src) For each ancestor frame f, summarize callee writes at the callsite: may_write(callsite, f) |= OR_{i in callee} may_write(i, f) Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-4-bcba9bc53ac9@gmail.com --- kernel/bpf/liveness.c | 207 ++++++++++++------ .../selftests/bpf/progs/verifier_live_stack.c | 38 +++- 2 files changed, 168 insertions(+), 77 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index d4089a488784..e18d5d86c301 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -23,6 +23,7 @@ enum { FM_MAY_READ, /* stack slots that may be read by this instruction */ FM_MUST_WRITE, /* stack slots written by this instruction */ FM_LIVE_BEFORE, /* stack slots that may be read by this insn and its successors */ + FM_MAY_WRITE, /* stack slots that may be written by this instruction */ FM_MASK_CNT, }; @@ -262,6 +263,12 @@ static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_ return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi); } +static int mark_stack_may_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MAY_WRITE, lo, hi); +} + /* * Mark every half-slot of @frame as possibly read by @insn_idx. This widens * the masks to the program's stack budget: a full read recorded at a narrower @@ -277,9 +284,16 @@ static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instanc env->stack_limit / BPF_HALF_REG_SIZE - 1); } -/* Accumulate @src, a mask @src_words wide, into may_read of @frame at @insn_idx */ -static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, - const unsigned long *src, u32 src_words) +static int mark_stack_may_write_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_may_write(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +/* Accumulate @src, a mask @src_words wide, into @kind mask of @frame at @insn_idx */ +static int mark_stack_mask(struct func_instance *instance, u32 frame, u32 insn_idx, u32 kind, + const unsigned long *src, u32 src_words) { u32 nbits = src_words * BITS_PER_LONG; struct frame_masks *fm; @@ -292,7 +306,7 @@ static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 i fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1)); if (!fm) return -ENOMEM; - dst = rel_mask(fm, relative_idx(instance, insn_idx), FM_MAY_READ); + dst = rel_mask(fm, relative_idx(instance, insn_idx), kind); /* @src has no bits set past @last, hence none past @fm->words either */ src_words = min(src_words, fm->words); for (w = 0; w < src_words; w++) @@ -300,6 +314,12 @@ static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 i return 0; } +static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, + const unsigned long *src, u32 src_words) +{ + return mark_stack_mask(instance, frame, insn_idx, FM_MAY_READ, src, src_words); +} + int bpf_jmp_offset(struct bpf_insn *insn) { u8 code = insn->code; @@ -624,16 +644,41 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, return env->tmp_str_buf; } +/* Print mask @kind of the instruction at relative index @i for every frame, if any bit is set. */ +static bool print_mask(struct bpf_verifier_env *env, struct func_instance *instance, int i, + const char *name, u32 kind) +{ + struct frame_masks *fm; + bool printed = false; + unsigned long *mask; + int frame; + u64 pos; + + pos = env->log.end_pos; + verbose(env, "%s", name); + for (frame = instance->depth; frame >= 0; --frame) { + fm = instance->frames[frame]; + if (!fm) + continue; + mask = rel_mask(fm, i, kind); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !printed, mask, fm->words)); + printed = true; + } + if (!printed) + bpf_vlog_reset(&env->log, pos); + return printed; +} + static void print_instance(struct bpf_verifier_env *env, struct func_instance *instance) { int start = env->subprog_info[instance->subprog].start; struct bpf_insn *insns = env->prog->insnsi; - struct frame_masks *fm; - unsigned long *mask; int len = instance->insn_cnt; - int insn_idx, frame, i; - bool has_use, has_def; u64 pos, insn_pos; + int insn_idx, i; + bool printed; if (!(env->log.level & BPF_LOG_LEVEL2)) return; @@ -642,41 +687,17 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i verbose(env, "%s:\n", fmt_instance(env, instance)); for (i = 0; i < len; i++) { insn_idx = start + i; - has_use = false; - has_def = false; pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); - pos = env->log.end_pos; - verbose(env, " use: "); - for (frame = instance->depth; frame >= 0; --frame) { - fm = instance->frames[frame]; - if (!fm) - continue; - mask = rel_mask(fm, i, FM_MAY_READ); - if (bitmap_empty(mask, frame_mask_bits(fm))) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, mask, fm->words)); - has_use = true; - } - if (!has_use) - bpf_vlog_reset(&env->log, pos); - pos = env->log.end_pos; - verbose(env, " def: "); - for (frame = instance->depth; frame >= 0; --frame) { - fm = instance->frames[frame]; - if (!fm) - continue; - mask = rel_mask(fm, i, FM_MUST_WRITE); - if (bitmap_empty(mask, frame_mask_bits(fm))) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, mask, fm->words)); - has_def = true; - } - if (!has_def) - bpf_vlog_reset(&env->log, has_use ? pos : insn_pos); + printed = false; + printed |= print_mask(env, instance, i, " use: ", FM_MAY_READ); + printed |= print_mask(env, instance, i, " def: ", FM_MUST_WRITE); + printed |= print_mask(env, instance, i, " may_def: ", FM_MAY_WRITE); + if (!printed) + bpf_vlog_reset(&env->log, insn_pos); verbose(env, "\n"); if (bpf_is_ldimm64(&insns[insn_idx])) i++; @@ -1437,12 +1458,14 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, } /* - * Record reads for every touched half-slot. A definite + * Record possible reads and writes for every touched half-slot. A definite * write requires full coverage, a known size, and a single possible offset. */ -static int record_stack_access_off(struct func_instance *instance, s64 fp_off, - struct arg_access_info info, u32 frame, u32 insn_idx) +static int record_stack_access_off(struct func_instance *instance, const struct arg_track *arg, + s64 off_idx, struct arg_access_info info, + u32 frame, u32 insn_idx) { + s64 fp_off = arg->off[off_idx]; s32 slot_hi, slot_lo; int err; @@ -1454,20 +1477,25 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, */ return 0; - if (info.may_read && info.size == U32_MAX) { - /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - return mark_stack_read(instance, frame, insn_idx, 0, slot_hi); - } + /* + * Read and may_write marks include partially covered slots. + * An unknown access size may reach from fp_off to the frame top. + */ + slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; + slot_lo = info.size == U32_MAX + ? 0 + : max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); if (info.may_read) { - /* Mark any touched slot as use */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - slot_lo = max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); err = mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); if (err) return err; } - if (info.must_write && info.size != U32_MAX) { + if (info.may_write) { + err = mark_stack_may_write(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.must_write && info.size != U32_MAX && arg->off_cnt == 1) { /* Mark only fully covered slots as def */ slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; slot_lo = max_t(s32, (-fp_off - info.size + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); @@ -1492,26 +1520,24 @@ static int record_stack_access(struct bpf_verifier_env *env, if (err) return err; } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, frame, insn_idx); + if (err) + return err; + } return 0; } - if (info.size != U32_MAX && info.must_write && arg->off_cnt != 1) - /* multi-offset write cannot set stack_def */ - return 0; - for (i = 0; i < arg->off_cnt; i++) { - err = record_stack_access_off(instance, arg->off[i], info, frame, insn_idx); + err = record_stack_access_off(instance, arg, i, info, frame, insn_idx); if (err) return err; } return 0; } -/* - * When a pointer is ARG_IMPRECISE, conservatively mark every frame in - * the bitmask as fully used. - */ +/* Record possible effects on every candidate frame of an imprecise pointer. */ static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance, - u32 mask, u32 insn_idx) + struct arg_access_info info, u32 mask, u32 insn_idx) { int depth = instance->depth; int f, err; @@ -1520,9 +1546,16 @@ static int record_imprecise(struct bpf_verifier_env *env, struct func_instance * if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read_all(env, instance, f, insn_idx); - if (err) - return err; + if (info.may_read) { + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, f, insn_idx); + if (err) + return err; + } } } return 0; @@ -1599,7 +1632,7 @@ static int record_load_store_access(struct bpf_verifier_env *env, if (ptr->frame >= 0 && ptr->frame <= depth) return record_stack_access(env, instance, ptr, info, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) - return record_imprecise(env, instance, ptr->mask, insn_idx); + return record_imprecise(env, instance, info, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } @@ -1627,6 +1660,9 @@ static int record_arg_access(struct bpf_verifier_env *env, err = mark_stack_read_all(env, instance, f, insn_idx); if (err) return err; + err = mark_stack_may_write_all(env, instance, f, insn_idx); + if (err) + return err; } return 0; } @@ -1636,7 +1672,7 @@ static int record_arg_access(struct bpf_verifier_env *env, if (frame >= 0 && frame <= depth) err = record_stack_access(env, instance, at, info, frame, insn_idx); else if (frame == ARG_IMPRECISE) - err = record_imprecise(env, instance, at->mask, insn_idx); + err = record_imprecise(env, instance, info, at->mask, insn_idx); return err; } @@ -1996,6 +2032,7 @@ static bool has_fp_args(struct arg_track *args) /* * Merge a freshly analyzed instance into the original. * may_read: union (any pass might read the slot). + * may_write: union (slots written on ANY pass). * must_write: intersection (only slots written on ALL passes are guaranteed). * live_before is recomputed by a subsequent update_instance() on @dst. * @@ -2034,18 +2071,50 @@ static int merge_instances(struct func_instance *dst, struct func_instance *src) for (i = 0; i < dst->insn_cnt; i++) { unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ); unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE); + unsigned long *dst_may_write = rel_mask(d, i, FM_MAY_WRITE); unsigned long *src_read = rel_mask(s, i, FM_MAY_READ); unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE); + unsigned long *src_may_write = rel_mask(s, i, FM_MAY_WRITE); for (w = 0; w < d->words; w++) { dst_read[w] |= w < s->words ? src_read[w] : 0; dst_write[w] &= w < s->words ? src_write[w] : 0; + dst_may_write[w] |= w < s->words ? src_may_write[w] : 0; } } } return 0; } +/* + * Fold a fully analyzed callee instance writes to upper frames as + * may_write marks at callsite in caller's frames. + */ +static int merge_may_write(struct func_instance *caller, struct func_instance *callee) +{ + DECLARE_BITMAP(acc, FRAME_HALF_SPIS); + u32 call_idx = callee->callsite; + struct frame_masks *fm; + u32 f, i, nbits; + int err; + + for (f = 0; f < callee->depth; f++) { + fm = callee->frames[f]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + bitmap_zero(acc, nbits); + for (i = 0; i < callee->insn_cnt; i++) + bitmap_or(acc, acc, rel_mask(fm, i, FM_MAY_WRITE), nbits); + if (bitmap_empty(acc, nbits)) + continue; + err = mark_stack_mask(caller, f, call_idx, FM_MAY_WRITE, acc, fm->words); + if (err) + return err; + } + return 0; +} + static struct func_instance *fresh_instance(struct func_instance *src) { struct func_instance *f; @@ -2161,6 +2230,9 @@ static int analyze_subprog(struct bpf_verifier_env *env, err = mark_stack_read_all(env, instance, f, idx); if (err) goto out_free; + err = mark_stack_may_write_all(env, instance, f, idx); + if (err) + goto out_free; } continue; } @@ -2213,6 +2285,11 @@ static int analyze_subprog(struct bpf_verifier_env *env, goto out_free; } } + + /* Summarize callee's writes to ancestor frames onto the callsite */ + err = merge_may_write(instance, callee_instance); + if (err) + goto out_free; } if (prev_instance) { diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 16b2b1e57534..3ba21430bb75 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -61,7 +61,11 @@ __naked void read_write_join(void) SEC("socket") __log_level(2) __msg("stack use/def subprog#0 must_write_not_same_slot (d0,cs0):") -__msg("6: (7b) *(u64 *)(r2 +0) = r0{{$}}") +/* + * 'r2 += r1' adds a scalar, so the offset is lost (off_cnt == 0): no def, + * but the write conservatively marks the whole frame as may_def. + */ +__msg("6: (7b) *(u64 *)(r2 +0) = r0 ; may_def: fp0-8..-{{(512|2048)}}") __msg("Live regs before insn:") __naked void must_write_not_same_slot(void) { @@ -106,9 +110,9 @@ __naked void must_write_not_same_type(void) SEC("socket") __log_level(2) -/* Callee writes fp[0]-8: stack_use at call site has slots 0,1 live */ +/* Callee writes fp[0]-8: the def is summarized as may_def at the call site */ __msg("stack use/def subprog#0 caller_stack_write (d0,cs0):") -__msg("2: (85) call pc+1{{$}}") +__msg("2: (85) call pc+1 ; may_def: fp0-8") __msg("stack use/def subprog#1 write_first_param (d1,cs2):") __msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8") __naked void caller_stack_write(void) @@ -804,9 +808,9 @@ void __kfunc_btf_root(void) */ SEC("socket") __success __log_level(2) -__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24{{$}}") -__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24{{$}}") -__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24{{$}}") +__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24 may_def: fp0-24{{$}}") +__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") +__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") __naked void kfunc_iter_stack_liveness(void) { asm volatile ( @@ -1008,7 +1012,8 @@ __naked void four_byte_read_upper_half(void) SEC("socket") __log_level(2) __msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") -__msg("1: (6a) *(u16 *)(r10 -4) = 0{{$}}") +/* 2-byte write only partially covers the upper half: may_def, but no def. */ +__msg("1: (6a) *(u16 *)(r10 -4) = 0 ; may_def: fp0-4h") __msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") __naked void two_byte_write_no_kill(void) { @@ -1355,9 +1360,12 @@ __naked void fp_spill_loses_precision_kills_liveness(void) */ SEC("socket") __log_level(2) -/* fp-8 live at call (callee conditionally writes → slot not killed) */ +/* + * fp-8 live at call: callee conditionally writes it, so the slot is not killed + * (no def), but the conditional write surfaces as may_def at the call site. + */ __msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") -__msg("4: (85) call pc+2{{$}}") +__msg("4: (85) call pc+2 ; may_def: fp0-8") __msg("5: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") __naked void conditional_stx_in_subprog(void) { @@ -2386,7 +2394,12 @@ __msg("subprog#2 write_first_read_second:") __msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") __msg("18: (79) r0 = *(u64 *)(r2 +0) // r1=fp0-8 r2=fp0-16{{$}}") __msg("stack use/def subprog#2 write_first_read_second (d2,cs15):") -__msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") +/* + * Shared across two callsites with swapped args (r1 is fp-8 on one pass, + * fp-16 on the other): must_write intersects to empty (no def), may_write + * unions to both slots. + */ +__msg("17: (7a) *(u64 *)(r1 +0) = 42 ; may_def: fp0-8 fp0-16") __msg("18: (79) r0 = *(u64 *)(r2 +0) ; use: fp0-8 fp0-16") __naked void shared_instance_must_write_overwrite(void) { @@ -2848,8 +2861,9 @@ static __used __naked void imprecise_dst_spill_join_sub(void) SEC("socket") __log_level(2) __msg("0: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") -__msg("1: (73) *(u8 *)(r10 -1) = r0{{$}}") -__msg("2: (6b) *(u16 *)(r10 -4) = r0{{$}}") +/* narrow stores define nothing, but they may write the half-slot they touch */ +__msg("1: (73) *(u8 *)(r10 -1) = r0 ; may_def: fp0-4h") +__msg("2: (6b) *(u16 *)(r10 -4) = r0 ; may_def: fp0-4h") __msg("3: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") __naked void narrow_store_defines_nothing(void) { From 0360ed8655cfe4ed25d5feb46c7f31addc59e63a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:23 -0700 Subject: [PATCH 393/660] bpf: Summarize may write stack slots in insn_aux_data SCEV needs may-write information to invalidate stack expressions after indirect writes. Liveness tracks this per function instance, but SCEV is not callchain-sensitive and needs a summary across calling contexts. For each instruction, union the current-frame may_write masks from all analyzed instances. Represent 8-byte slots as a summary of constituent halves. Store the summary in insn_aux_data and expose it via a helper. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-5-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 6 ++++++ kernel/bpf/liveness.c | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index eafe79929dfa..4cb36ee3b6f7 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -662,6 +662,11 @@ struct bpf_insn_aux_data { }; struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ + /* + * Per-instruction summary of stack slots in the current frame + * that this instruction may write to. + */ + DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ bool nospec; /* do not execute this instruction speculatively */ @@ -1742,6 +1747,7 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index e18d5d86c301..ec9c8beef556 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -718,6 +718,45 @@ static int cmp_instances(const void *pa, const void *pb) return 0; } +/* OR the 8-byte slots touched by a half-slot (4-byte) mask into @slots. */ +static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, u32 nbits) +{ + u32 slot; + + for (slot = 0; slot < MAX_BPF_STACK_SLOTS && slot * 2 + 1 < nbits; slot++) + if (test_bit(slot * 2, mask) || test_bit(slot * 2 + 1, mask)) + __set_bit(slot, slots); +} + +/* + * Precompute, for each instruction, the OR of may_write masks over its top + * frame across all func_instances reaching it, stash it in the insn_aux_data. + */ +static void compute_may_write_masks(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_liveness *liveness = env->liveness; + struct func_instance *instance; + struct frame_masks *fm; + u32 nbits; + int bkt, i; + + hash_for_each(liveness->func_instances, bkt, instance, hl_node) { + fm = instance->frames[instance->depth]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + for (i = 0; i < instance->insn_cnt; i++) + half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask, + rel_mask(fm, i, FM_MAY_WRITE), nbits); + } +} + +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx) +{ + return env->insn_aux_data[insn_idx].may_write_mask; +} + /* print use/def slots for all instances ordered by callsite first, then by depth */ static int print_instances(struct bpf_verifier_env *env) { @@ -2352,6 +2391,8 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env) goto out; } + compute_may_write_masks(env); + if (env->log.level & BPF_LOG_LEVEL2) err = print_instances(env); From 30175f5d9b1a82e6664485bfe9e50e4dcaa6f2ac Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:24 -0700 Subject: [PATCH 394/660] bpf: Summarize live stack slots in insn_aux_data SCEV needs live stack information in order to skip tracking dead slots. Liveness tracks this per function instance, but SCEV is not callchain-sensitive and needs a summary across calling contexts. For each instruction, union the current-frame liveness masks from all analyzed function instances. Represent 8-byte slots as a summary of constituent halves. Store the summary in insn_aux_data. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-6-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/liveness.c | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 4cb36ee3b6f7..5b615083ffa9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -667,6 +667,7 @@ struct bpf_insn_aux_data { * that this instruction may write to. */ DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); + DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ bool nospec; /* do not execute this instruction speculatively */ diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index ec9c8beef556..004c1f48a065 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -729,8 +729,9 @@ static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, } /* - * Precompute, for each instruction, the OR of may_write masks over its top - * frame across all func_instances reaching it, stash it in the insn_aux_data. + * Precompute, for each instruction, the OR of may_write and live_before masks + * over its top frame across all func_instances reaching it, stash them in the + * insn_aux_data. */ static void compute_may_write_masks(struct bpf_verifier_env *env) { @@ -746,9 +747,12 @@ static void compute_may_write_masks(struct bpf_verifier_env *env) if (!fm) continue; nbits = frame_mask_bits(fm); - for (i = 0; i < instance->insn_cnt; i++) + for (i = 0; i < instance->insn_cnt; i++) { half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask, rel_mask(fm, i, FM_MAY_WRITE), nbits); + half_spis_to_slots(aux[instance->subprog_start + i].live_stack_before, + rel_mask(fm, i, FM_LIVE_BEFORE), nbits); + } } } From c6477be1580445be005c8c249f00fc44eeca3190 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:25 -0700 Subject: [PATCH 395/660] bpf: Summarize regs that may hold a frame pointer in insn_aux_data SCEV needs to know which registers might be stack pointers at a particular instruction. This information is used to avoid widening registers if that would hinder further verification. See the patch [1] for additional information. liveness.c:compute_subprog_args() already tracks this information. This commit modifies it to save the information in insn_aux_data for further usage. [1] "bpf: avoid widening registers that hinder exact stack-slot tracking" Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-7-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/liveness.c | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5b615083ffa9..d31c1a500958 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -668,6 +668,7 @@ struct bpf_insn_aux_data { */ DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); + u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ bool nospec; /* do not execute this instruction speculatively */ diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 004c1f48a065..b9b76684906e 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1896,6 +1896,17 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } } +static void record_stack_ptrs(struct bpf_verifier_env *env, int idx, struct arg_track *at_in) +{ + u16 r, mask = 0; + + for (r = 0; r < MAX_BPF_REG; r++) + if (arg_is_fp(&at_in[r])) + mask |= BIT(r); + + env->insn_aux_data[idx].stack_ptrs |= mask; +} + /* * Compute arg tracking dataflow for a single subprog. * Runs forward fixed-point with arg_track_xfer(), then records @@ -2045,6 +2056,8 @@ redo: snap->slots = nslots; memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at)); } + + record_stack_ptrs(env, idx, at_in[i]); } info->at_in = at_in; From 2d903f95721a0e0084c8b671e09c1b9425b1a448 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:26 -0700 Subject: [PATCH 396/660] bpf: Record write effects for atomic operations in liveness.c Atomic read-modify-write instructions are recorded as reads only, omitting their stack write effects from the may_write masks needed by SCEV. Record both read and write accesses for atomic RMW operations. Keep LOAD_ACQ read-only and STORE_REL write-only. Share the precise/imprecise stack-access dispatch so both accesses use the same address handling. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-8-bcba9bc53ac9@gmail.com --- kernel/bpf/liveness.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index b9b76684906e..ce74e3f16eef 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1634,14 +1634,22 @@ static int record_load_store_access(struct bpf_verifier_env *env, break; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (insn->imm == BPF_STORE_REL) - write = true; - else - read = true; - if (insn->imm == BPF_LOAD_ACQ) + switch (insn->imm) { + case BPF_LOAD_ACQ: ptr = &at[insn->src_reg]; - else + read = true; + break; + case BPF_STORE_REL: ptr = &at[insn->dst_reg]; + write = true; + break; + default: + /* ADD/AND/OR/XOR(+FETCH), XCHG, CMPXCHG */ + ptr = &at[insn->dst_reg]; + read = true; + write = true; + break; + } } else { ptr = &at[insn->dst_reg]; write = true; From 13cffb88318085117c988106d4f5e439646f3ba6 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:27 -0700 Subject: [PATCH 397/660] bpf: Fix arg_track_xfer() for atomic stack spills and fills Track spilled stack pointers through 64-bit atomic accesses: - BPF_STORE_REL spills the source register. - BPF_LOAD_ACQ restores the spilled value into dst_reg. - BPF_XCHG and BPF_{ADD,AND,OR,XOR} | BPF_FETCH restore it into src_reg. - BPF_CMPXCHG restores it into R0. This prevents missing stack accesses through pointers recovered by these instructions, matching the main verifier's spill/fill behavior. Fixes: bf0c571f7feb ("bpf: introduce forward arg-tracking dataflow analysis") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-9-bcba9bc53ac9@gmail.com --- kernel/bpf/liveness.c | 91 ++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index ce74e3f16eef..52c39bbee6b1 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1357,6 +1357,29 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth))); } +/* + * Model 8-byte reads through @reg, including load-acquire + * and the old value fetched by atomic operations. + */ +static struct arg_track arg_track_deref(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct arg_track *at_out, int reg, + struct arg_track *at_stack_out, u32 nslots, + int depth, u32 *callsites) +{ + struct arg_track *ptr = &at_out[reg]; + + if (can_be_local_fp(depth, reg, ptr)) + return fill_from_stack(insn, at_out, reg, at_stack_out, nslots, depth); + if (ptr->frame >= 0 && ptr->frame < depth) { + struct spill_snapshot *parent = env->callsite_at_stack[callsites[ptr->frame]]; + + return fill_from_stack(insn, at_out, reg, parent->at, parent->slots, ptr->frame); + } + if (ptr->frame == ARG_IMPRECISE && !(ptr->mask & BIT(depth)) && ptr->mask) + return *ptr; + return (struct arg_track){ .frame = ARG_NONE }; +} + /* * Pure dataflow transfer function for arg_track state. * Updates at_out[] based on how the instruction modifies registers. @@ -1438,56 +1461,42 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, for (r = BPF_REG_0; r <= BPF_REG_5; r++) at_out[r] = none; } else if (class == BPF_LDX) { - u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src); - - /* - * Reload from callee stack: if src is current-frame FP-derived - * and the load is an 8-byte BPF_MEM, try to restore the spill - * identity. For imprecise sources fill_from_stack() returns - * ARG_IMPRECISE (off_cnt == 0). - */ - if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, nslots, - depth); - } else if (src->frame >= 0 && src->frame < depth && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - struct spill_snapshot *parent = - env->callsite_at_stack[callsites[src->frame]]; - - *dst = fill_from_stack(insn, at_out, insn->src_reg, - parent->at, parent->slots, src->frame); - } else if (src->frame == ARG_IMPRECISE && - !(src->mask & BIT(depth)) && src->mask && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - /* - * Imprecise src with only parent-frame bits: - * conservative fallback. - */ - *dst = *src; - } else { + /* Only an 8-byte BPF_MEM load can be a fill, see arg_track_deref(). */ + if (BPF_MODE(insn->code) == BPF_MEM && BPF_SIZE(insn->code) == BPF_DW) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else *dst = none; - } } else if (class == BPF_LD && BPF_MODE(insn->code) == BPF_IMM) { *dst = none; } else if (class == BPF_STX) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool dst_is_local_fp; + bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); + u8 mode = BPF_MODE(insn->code); - /* Track spills to current-frame FP-derived callee stack */ - dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); - if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) - spill_to_stack(insn, at_out, insn->dst_reg, - at_stack_out, nslots, src, sz); - - if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ) - clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, nslots, sz); + if (mode == BPF_MEM || (mode == BPF_ATOMIC && insn->imm == BPF_STORE_REL)) { + if (dst_is_local_fp) + spill_to_stack(insn, at_out, insn->dst_reg, + at_stack_out, nslots, src, sz); + } else if (mode == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { + if (sz == 8) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else + *dst = none; + } else if (mode == BPF_ATOMIC) { + /* BPF_XCHG, BPF_CMPXCHG or BPF_{ADD,AND,OR,XOR} | [BPF_FETCH] */ + struct arg_track old = none; r = bpf_atomic_load_reg(insn); + if (r >= 0 && sz == 8) + old = arg_track_deref(env, insn, at_out, insn->dst_reg, + at_stack_out, nslots, depth, callsites); + if (dst_is_local_fp) + clear_stack_for_all_offs(insn, at_out, insn->dst_reg, + at_stack_out, nslots, sz); if (r >= 0) - at_out[r] = none; + at_out[r] = old; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); From 8212c1c2986dfd06498558f672ff3ec964b924c5 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:28 -0700 Subject: [PATCH 398/660] bpf: Derive global subprog stack liveness from BTF signatures Stack liveness currently analyzes global callees in the context of their callers. The resulting read/write marks describe only the current body, but freplace can replace it after the caller has been verified. A no-op callee taking a stack pointer can be replaced with one writing through that pointer, leaving the call's may_write marks incomplete. SCEV could then preserve a stack counter recurrence across a call that resets it, invalidating a loop termination proof. Instead, derive global call effects from BTF argument types instead, same is it done on the main verification pass. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-10-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 3 ++ kernel/bpf/liveness.c | 28 +++++++++++---- kernel/bpf/verifier.c | 66 ++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d31c1a500958..05e6f091ad94 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1744,6 +1744,9 @@ bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct arg_access_info bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); int bpf_stack_liveness_init(struct bpf_verifier_env *env); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 52c39bbee6b1..a359e457e6c0 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1715,6 +1715,8 @@ static int record_arg_access(struct bpf_verifier_env *env, info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { info = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else if (bpf_pseudo_call(insn)) { + info = bpf_global_subprog_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { err = mark_stack_read_all(env, instance, f, insn_idx); @@ -1736,7 +1738,7 @@ static int record_arg_access(struct bpf_verifier_env *env, return err; } -/* Record stack access for a given 'at' state of helper/kfunc 'insn' */ +/* Record stack access for a given 'at' state of helper/kfunc/global subprog 'insn' */ static int record_call_access(struct bpf_verifier_env *env, struct func_instance *instance, struct arg_track *at, @@ -1744,12 +1746,19 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, arg_slot_cnt = 5; + int r, err, callee, arg_slot_cnt = 5; - if (bpf_pseudo_call(insn)) - return 0; - - if (bpf_is_callx(insn)) + if (bpf_pseudo_call(insn)) { + /* + * analyze_subprog() handles static callees. For global callees, + * derive effects from argument types so may_write marks remain + * conservative under freplace. + */ + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0 || !bpf_subprog_is_global(env, callee)) + return 0; + arg_slot_cnt = MAX_BPF_FUNC_ARGS; + } else if (bpf_is_callx(insn)) /* * The callee is not known statically. Assume that all arg * slots are passed and let record_arg_access() conservatively @@ -2282,7 +2291,12 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) { target = idx + insn->imm + 1; callee = bpf_find_subprog(env, target); - if (callee < 0) + /* + * Global subprograms can be freplaced, so conservatively derive + * their stack effects from their BTF signatures instead of + * analyzing them in the context of their callers. + */ + if (callee < 0 || bpf_subprog_is_global(env, callee)) continue; /* Build entry args: R1-R5 and stack args from at_in at call site */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8987c9a69d10..a5456a6e2694 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14571,6 +14571,72 @@ out: return info; } +/* + * Called before dead code elimination, some subprograms may be genuinely unreachable. + * Suppress diagnostics so callers can use a conservative result on error. + * The main pass would report errors for reachable subprograms. + */ +static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_prog *prog = env->prog; + u32 log_level = env->log.level; + int err; + + if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id) + return false; + + env->log.level = 0; + err = btf_prepare_func_args(env, subprog); + env->log.level = log_level; + return !err; +} + +/* + * Derive stack effects for a global subprog's argument slot @arg (0-based) + * from its declared type, matching the main pass's argument checks. + */ +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) +{ + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; + struct bpf_subprog_info *sub; + int callee; + + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0) + return info; + sub = subprog_info(env, callee); + if (!try_btf_prepare_func_args(env, callee)) + return info; + + switch (base_type(sub->args[arg].arg_type)) { + case ARG_UNUSED: /* Not an argument slot. */ + case ARG_ANYTHING: /* A scalar, the callee can't dereference it. */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_BTF_ID: + /* + * Trusted arguments cannot be stack pointers. + * Untrusted arguments use ARG_IGNORE: + * probe reads need no caller stack state, writes are prohibited. + */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_MEM: + /* mem_size is zero for untrusted pointers of unknown size. */ + if (sub->args[arg].mem_size) { + info.size = sub->args[arg].mem_size; + info.must_write = true; + } + return info; + default: + return info; + } +} + /* check special kfuncs and return: * 1 - not fall-through to 'else' branch, continue verification * 0 - fall-through to 'else' branch From 147400e9c27c0c8df3d96568e6634568e968842b Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:29 -0700 Subject: [PATCH 399/660] bpf: Compute parameters liveness in bpf_compute_live_registers() This computes exact liveness for the arguments of a call to a subprogram instead of assuming that r1..r5 are read: - For static subprograms compute use registers at the bpf_compute_live_registers() fixpoint phase. - For global subprograms use BTF information. Add a separate pass doing try_btf_prepare_func_args() in order for bpf_subprog_info->arg_slot_cnt to be available during bpf_compute_live_registers(). The conservative liveness marks hindered SCEV widening logic on a set of production programs. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-11-bcba9bc53ac9@gmail.com --- kernel/bpf/liveness.c | 31 +++++++++++++++++-- kernel/bpf/verifier.c | 24 ++++++++++++++ .../selftests/bpf/prog_tests/exceptions.c | 2 +- .../bpf/progs/compute_live_registers.c | 12 +++---- .../selftests/bpf/progs/verifier_live_stack.c | 2 +- 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index a359e457e6c0..8d8428ccc505 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2472,6 +2472,12 @@ static inline u32 mask_widen(u32 m) { return m | (m << 16); } static inline u16 mask_lo(u32 m) { return (u16)m; } static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } +/* Index of the subprogram called by pseudo call @insn */ +static int callee_subprog(struct bpf_verifier_env *env, const struct bpf_insn *insn) +{ + return bpf_find_subprog(env, insn - env->prog->insnsi + insn->imm + 1); +} + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, @@ -2489,6 +2495,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); const u32 r2 = reg64_mask(BPF_REG_2); + int subprog, reg_args; u32 def = 0; u32 use = U32_MAX; @@ -2613,8 +2620,18 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_CALL: def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); - if (bpf_get_call_summary(env, insn, &cs)) + if (bpf_get_call_summary(env, insn, &cs)) { use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); + } else if (bpf_pseudo_call(insn)) { + /* a static callee's reads are added in bpf_compute_live_registers() */ + subprog = callee_subprog(env, insn); + if (bpf_subprog_is_global(env, subprog)) { + reg_args = min_t(u8, env->subprog_info[subprog].arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + use = GENMASK(reg_args, 1); + } else { + use = 0; + } + } def = mask_widen(def); use = mask_widen(use); /* callx reads the address of the callee from dst_reg */ @@ -2699,11 +2716,21 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_iarray *succ; u32 new_out = 0; u32 new_in = 0; + u32 use; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) new_out |= state[succ->items[s]].in; - new_in = (new_out & ~live->def) | live->use; + use = live->use; + /* + * For a call to a static subprogram include the callee's + * entry 'in' as computed so far. + */ + if (bpf_pseudo_call(&insns[insn_idx]) && + !bpf_subprog_is_global(env, callee_subprog(env, &insns[insn_idx]))) + use |= state[insn_idx + insns[insn_idx].imm + 1].in & + mask_widen(GENMASK(BPF_REG_5, BPF_REG_1)); + new_in = (new_out & ~live->def) | use; if (new_out != live->out || new_in != live->in) { live->in = new_in; live->out = new_out; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a5456a6e2694..50cb564be9f7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10913,6 +10913,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s struct bpf_call_arg_meta meta; struct bpf_func_proto *fn; int ret, err; + u32 i; memset(&meta, 0, sizeof(meta)); meta.btf = btf; @@ -10939,6 +10940,15 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s meta.fn = fn; meta.func_proto = func_proto; + /* + * Check if all register arguments declared in BTF are initialized. + * If they aren't the BTF is unreliable. + */ + if (!bpf_subprog_is_global(env, subprog)) { + for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) + if (caller->regs[BPF_REG_1 + i].type == NOT_INIT) + return -EINVAL; + } return check_func_args(env, &meta, env->insn_idx); } @@ -14591,6 +14601,18 @@ static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return !err; } +/* Prepare argument metadata for liveness; the main program is handled separately. */ +static void prepare_subprog_args(struct bpf_verifier_env *env) +{ + int i; + + for (i = 1; i < env->subprog_cnt; i++) { + if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i)) + /* Keep all argument registers live if the prototype is unavailable. */ + env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS; + } +} + /* * Derive stack effects for a global subprog's argument slot @arg (0-based) * from its declared type, matching the main pass's argument checks. @@ -22836,6 +22858,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + prepare_subprog_args(env); + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 3705db18c1bb..cea01d638f6f 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -322,7 +322,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "subprog"), "set_attach_target")) goto done; - }), "subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> throwing subprog")) /* non-throwing extension -> throwing subprog : BAD (!global) */ diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index 941b65fe7c07..b274a20ed7bd 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -416,11 +416,11 @@ static __used __naked int aux1(void) SEC("socket") __log_level(2) -__msg("0: ....45.... (b7) r1 = 1") -__msg("1: .1..45.... (b7) r2 = 2") -__msg("2: .12.45.... (b7) r3 = 3") -/* Conservative liveness for subprog parameters. */ -__msg("3: .12345.... (85) call pc+2") +__msg("0: .......... (b7) r1 = 1") +__msg("1: .1........ (b7) r2 = 2") +__msg("2: .12....... (b7) r3 = 3") +/* A static callee reads what its code reads: aux1 reads r1 and r2. */ +__msg("3: .12....... (85) call pc+2") __msg("4: .......... (b7) r0 = 0") __msg("5: 0......... (95) exit") __msg("6: .12....... (bf) r0 = r1") @@ -452,7 +452,7 @@ SEC("socket") /* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ __load_if_JITed() __log_level(2) -__msg("0: .12345.... (85) call pc+2") +__msg("0: .......... (85) call pc+2") __msg("1: ..2....... (bf) r0 = r2") __msg("2: 0......... (95) exit") __msg("3: .......... (b7) r0 = 1") diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 3ba21430bb75..3abe08cc6595 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -1470,7 +1470,7 @@ static __used __naked void inner_reader(void) SEC("socket") __log_level(2) /* fp-8 NOT live at call: callee writes before reading (parent_def kills it) */ -__msg("2: .12345.... (85) call pc+") +__msg("2: .1........ (85) call pc+") __naked void callee_must_write_before_read(void) { asm volatile ( From 4072d7f56b44e1479b97239711c71de5b8106020 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:30 -0700 Subject: [PATCH 400/660] bpf: Add tnum_alignment() Used by the SCEV widening logic later in the series. The verifier checks alignment for some memory accesses. When SCEV widens a pointer or an index variable, it must retain alignment information in order to allow array access. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-12-bcba9bc53ac9@gmail.com --- include/linux/tnum.h | 2 ++ kernel/bpf/tnum.c | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/include/linux/tnum.h b/include/linux/tnum.h index ca2cfec8de08..16682248f963 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -134,4 +134,6 @@ static inline bool tnum_subreg_is_const(struct tnum a) /* Returns the smallest member of t larger than z */ u64 tnum_step(struct tnum t, u64 z); +u32 tnum_alignment(struct tnum a); + #endif /* _LINUX_TNUM_H */ diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index ec9c310cf5d7..1f620facd1b7 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -317,3 +317,14 @@ u64 tnum_step(struct tnum t, u64 z) inc = (filled + 1) & t.mask; return t.value | inc; } + +/* + * Return the number of trailing bits known to be zero in a tnum. + * Return 64 for a tnum representing zero. + */ +u32 tnum_alignment(struct tnum a) +{ + u64 v = a.value | a.mask; + + return v ? __ffs64(v) : 64; +} From a8e9a53f6df65f0811424dc419127dc7a9e646c4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:31 -0700 Subject: [PATCH 401/660] bpf: Add cnum{32,64}_union() Add cnum32_union() and cnum64_union() to compute a smallest circular range containing both inputs. Do so by enumerating the following configurations in a rotated frame (a.base at the origin): 0 UT_MAX |---------------------------------------------------| [= a ==============================] | [= b tail =] [= b main ===================> [= union ===========================================] 0 UT_MAX |---------------------------------------------------| [= a =====================] | [= b tail ======] [= b main =====> [= union tail ============] [= union main => 0 UT_MAX |---------------------------------------------------| [= a =======] | [= b tail =============] [= b main =====> [= union tail =========] [= union main => 0 UT_MAX |---------------------------------------------------| [= a =========================] | | [= b ====================] | [= union ==================================] | 0 UT_MAX |---------------------------------------------------| [= a =====================================] | | [= b ========] | [= union =================================] | 0 UT_MAX |---------------------------------------------------| [= a ============] | | [= b =======] | Two possible covering arcs: [= ab ======================================] | [= ba tail ======] [= ba main =========> Pick the smaller one. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-13-bcba9bc53ac9@gmail.com --- include/linux/cnum.h | 2 ++ kernel/bpf/cnum_defs.h | 44 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/include/linux/cnum.h b/include/linux/cnum.h index 49b7d0c7645d..ddf4726841e6 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h @@ -39,6 +39,7 @@ u32 cnum32_umin(struct cnum32 cnum); u32 cnum32_umax(struct cnum32 cnum); s32 cnum32_smin(struct cnum32 cnum); s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_union(struct cnum32 a, struct cnum32 b); struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); @@ -65,6 +66,7 @@ u64 cnum64_umin(struct cnum64 cnum); u64 cnum64_umax(struct cnum64 cnum); s64 cnum64_smin(struct cnum64 cnum); s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_union(struct cnum64 a, struct cnum64 b); struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 30685e43de04..05b17a8b4814 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -198,6 +198,50 @@ static inline struct cnum_t FN(normalize)(struct cnum_t cnum) return cnum; } +/* + * Return a smallest arc containing both 'a' and 'b'. + * Break equal-size ties by choosing the smaller base. + */ +struct cnum_t FN(union)(struct cnum_t a, struct cnum_t b) +{ + struct cnum_t b1, ab, ba; + ut end; + + if (FN(is_empty)(a)) + return b; + if (FN(is_empty)(b)) + return a; + + /* + * Rotate so that a1.base == 0 and a1.end == a.size. + * Normalize b1 to preserve the full-circle representation. + */ + b1 = FN(normalize)((struct cnum_t){ b.base - a.base, b.size }); + end = max(a.size, (ut)(b1.base + b1.size)); + + if (FN(urange_overflow)(b1)) { + /* a1 reaches b1's main arc: together they cover the circle. */ + if (b1.base <= a.size) + return (struct cnum_t){ 0, UT_MAX }; + + /* Extend b1's tail through a1's end, then rotate back. */ + return FN(normalize)((struct cnum_t){ b.base, end - b1.base }); + } + + /* ab, rotated back, covers both nonwrapping arcs. */ + ab = (struct cnum_t){ a.base, end }; + if (b1.base <= a.size) + return FN(normalize)(ab); + + /* The arcs are disjoint; ba is the other possible covering arc. */ + ba = (struct cnum_t){ b.base, a.size - b1.base }; + if (ba.size < ab.size || + (ba.size == ab.size && ba.base < ab.base)) + ab = ba; + + return FN(normalize)(ab); +} + struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b) { if (FN(is_empty)(a) || FN(is_empty)(b)) From 78e61c01a760c9c7ac7f3f3646249a94d9d33ec0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:32 -0700 Subject: [PATCH 402/660] bpf: Add cnum64_intersect_linear() Add cnum64_intersect_linear(), which intersects a cnum64 interval with the set of integers congruent to 'base' modulo 'step', i.e. the points base + step * k. It tightens the interval by rounding its signed minimum up and its signed maximum down to the nearest such point, and returns an empty cnum when no point falls within the interval. The arithmetic is done on the signed bounds [smin, smax]. The line is defined over the integers, while a cnum64 is an arc on the u64 circle, so a u64-modular computation would misplace the line for negative values whenever 'step' does not divide 2^64. Working from the signed bounds keeps the residues consistent, e.g. it can reason about a range like [-9, 3] with step 3. Also add imod() to bpf_verifier.h: a mathematical modulo returning an always-non-negative residue (unlike C's '%', which follows the sign of the dividend). It is used by cnum64_intersect_linear() and by the base/step tracking added in subsequent patches. This is a building block for tracking scalar registers whose value is known to lie on a line base + step * k. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-14-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 14 ++++++++++++++ include/linux/cnum.h | 1 + kernel/bpf/cnum.c | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 05e6f091ad94..0422d4a1be01 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -9,6 +9,20 @@ #include /* for MAX_BPF_STACK */ #include #include +#include /* for div_s64_rem() */ + +/* + * Mathematical modulo, the residue of 'v' modulo 'step', normalized to [0, step). + * This differs from C's '%' operator, which truncates the quotient toward zero + * and so returns a remainder with the sign of the dividend (e.g. -1 % 3 == -1, not 2). + */ +static inline u16 imod(s64 v, u16 step) +{ + s32 rem; + + div_s64_rem(v, step, &rem); + return rem < 0 ? rem + step : rem; +} /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit diff --git a/include/linux/cnum.h b/include/linux/cnum.h index ddf4726841e6..49160318adf9 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h @@ -80,5 +80,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step); #endif /* _LINUX_CNUM_H */ diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c index 86142cb2aee5..8ec5187d7c75 100644 --- a/kernel/bpf/cnum.c +++ b/kernel/bpf/cnum.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include /* for imod() */ #define T 32 #include "cnum_defs.h" @@ -118,3 +119,38 @@ struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b) } return t; } + +/* Intersect 'in' with the set of integers defined by equation 'base + step * k'. */ +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step) +{ + s64 smin = cnum64_smin(in); + s64 smax = cnum64_smax(in); + s64 lo, hi; + u16 d; + + if (step <= 1 || cnum64_is_empty(in) || cnum64_srange_overflow(in)) + return in; + /* + * Round smin up to the next value congruent to 'base' modulo 'step', + * i.e. increase smin by d = (base - smin) mod step: + * + * |<---- d ---->| + * |-----------|=============|... + * base+step*k smin base+step*(k+1) + */ + d = imod(base - imod(smin, step), step); + if ((u64)smax - (u64)smin < d) + return CNUM64_EMPTY; + lo = smin + d; + /* + * Round smax down to the previous value congruent to 'base' modulo 'step', + * i.e. decrease smax by d = (smax - base) mod step: + * + * |<--- d --->| + * ...|===========|-------------| + * base+step*k smax base+step*(k+1) + */ + d = imod(imod(smax, step) - base, step); + hi = smax - d; + return cnum64_from_srange(lo, hi); +} From 685ac494443e70b732a234c3effb446631d2270d Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:33 -0700 Subject: [PATCH 403/660] bpf: Expose bpf_rev_opcode() and bpf_flip_opcode() Expose the existing helpers verifier.c:rev_opcode() and verifier.c:flip_opcode() as bpf_rev_opcode() and bpf_flip_opcode() and declare them in bpf_verifier.h. The helpers are used by SCEV logic while analyzing loop conditions. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-15-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 3 +++ kernel/bpf/verifier.c | 16 +++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 0422d4a1be01..5ec9495d42b1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1875,4 +1875,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); +int bpf_flip_opcode(u32 opcode); +u8 bpf_rev_opcode(u8 opcode); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 50cb564be9f7..e317cdae749e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17333,8 +17333,6 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32); -static u8 rev_opcode(u8 opcode); - /* * Learn more information about live branches by simulating refinement on both branches. * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means @@ -17343,7 +17341,7 @@ static u8 rev_opcode(u8 opcode); static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32) { /* Fallthrough (FALSE) branch */ - regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32); + regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32); reg_bounds_sync(&env->false_reg1); reg_bounds_sync(&env->false_reg2); /* @@ -17529,7 +17527,7 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s return simulate_both_branches_taken(env, opcode, is_jmp32); } -static int flip_opcode(u32 opcode) +int bpf_flip_opcode(u32 opcode) { /* How can we transform "a b" into "b a"? */ static const u8 opcode_flip[16] = { @@ -17560,7 +17558,7 @@ static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, pkt = dst_reg; } else if (dst_reg->type == PTR_TO_PACKET_END) { pkt = src_reg; - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); } else { return -1; } @@ -17615,7 +17613,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); } /* and ensure that reg2 is a constant */ @@ -17649,7 +17647,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* Opcode that corresponds to a *false* branch condition. * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */ -static u8 rev_opcode(u8 opcode) +u8 bpf_rev_opcode(u8 opcode) { switch (opcode) { case BPF_JEQ: return BPF_JNE; @@ -17684,7 +17682,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state case BPF_JGT: case BPF_JSGE: case BPF_JSGT: - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); break; default: @@ -17751,7 +17749,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; - case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ + case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */ if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) From 041d7c85c45d86520e15b2068f5b90fd5a76861d Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:34 -0700 Subject: [PATCH 404/660] bpf: Expose bpf_grow_stack_state() When windening SCEV expressions like *fp-8 -> (any *fp-8 *fp-16), fp-8 might be not yet allocated. Expose a utility function such that widening logic can handle this. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-16-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 3 ++- kernel/bpf/verifier.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5ec9495d42b1..ab86b75888c3 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -522,7 +522,7 @@ static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) /* * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). - * The caller must ensure spi < bpf_stack_nr_slots(frame), see grow_stack_state(). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state(). */ static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) { @@ -1297,6 +1297,7 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_ void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e317cdae749e..47a632a6749a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1456,7 +1456,7 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n) /* Possibly update state->allocated_stack to be at least size bytes. Also * possibly update the function's high-water mark in its bpf_subprog_info. */ -static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { size_t old_n = bpf_stack_nr_slots(state), n; @@ -6705,7 +6705,7 @@ static int check_stack_access_within_bounds( /* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ - return grow_stack_state(env, state, -min_off /* size */); + return bpf_grow_stack_state(env, state, -min_off /* size */); } static bool get_func_retval_range(struct bpf_prog *prog, From f21dcf26986c9aafe2ec12f80c440da7394dfb21 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:35 -0700 Subject: [PATCH 405/660] bpf: Expose bpf_verbose_snum() For use in SCEV logging to make large widened bounds more readable. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-17-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/log.c | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ab86b75888c3..b4396867c6f3 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1165,6 +1165,7 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index d850a7863d2e..87d137734db1 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -527,7 +527,7 @@ static void verbose_unum(struct bpf_verifier_env *env, u64 num) verbose(env, "%#llx", num); } -static void verbose_snum(struct bpf_verifier_env *env, s64 num) +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) { if (is_snum_decimal(num)) verbose(env, "%lld", num); @@ -609,7 +609,7 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } if (m1->name[0] == 's') - verbose_snum(env, m1->val); + bpf_verbose_snum(env, m1->val); else verbose_unum(env, m1->val); } @@ -632,7 +632,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); return; } @@ -643,7 +643,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value + reg->delta); + bpf_verbose_snum(env, reg->var_off.value + reg->delta); return; } } @@ -667,7 +667,7 @@ static void print_reg_state(struct bpf_verifier_env *env, } if (t != SCALAR_VALUE && reg->delta) { verbose_a("off="); - verbose_snum(env, reg->delta); + bpf_verbose_snum(env, reg->delta); } if (type_is_pkt_pointer(t)) { verbose_a("r="); @@ -683,7 +683,7 @@ static void print_reg_state(struct bpf_verifier_env *env, /* a pointer register with fixed offset */ if (reg->var_off.value) { verbose_a("imm="); - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); } } else { print_scalar_ranges(env, reg, &sep); From 366b4c68d83042a45574272eddff8a76c9c25164 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:36 -0700 Subject: [PATCH 406/660] bpf: Allow subrange relations for PTR_TO_STACK in regsafe() regsafe() currently requires an exact match for PTR_TO_STACK registers. This prevents an explored state from pruning a current state whose variable-offset range is a subset of the explored range. This change allows a checkpoint containing a widened stack pointer to cover narrower offsets reached on subsequent loop iterations, for example [-56, -8] within [-64, -8]. Merge the PTR_TO_STACK case with PTR_TO_MEM and others, as the set of necessary checks is almost identical. The only difference is a map_uid check, which is not needed for PTR_TO_STACK. All locations that produce PTR_TO_STACK reset map_uid to zero. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-18-bcba9bc53ac9@gmail.com --- kernel/bpf/states.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 18bf7b660c2f..345079cf64d3 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -610,6 +610,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_MEM: case PTR_TO_BUF: case PTR_TO_TP_BUFFER: + case PTR_TO_STACK: /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. */ @@ -643,8 +644,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); - case PTR_TO_STACK: - return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: From 0892ef2a7ef31e8dcf16706f3163aa25694da88c Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:37 -0700 Subject: [PATCH 407/660] bpf: Representation for intervals with steps Extend scalar register tracking with a linear "base + step * k" description: each scalar carries two u16 fields, base and step (invariant: base < step), recording that its value is some point on the line base + step * k. A fresh scalar starts as base=0, step=1, i.e. any integer. This lets the verifier reason about strided values, e.g. an index multiplied by an element size, which neither the min/max bounds nor the tnum can capture for a non-power-of-2 stride. Such reasoning is necessary to allow array access with a variable offset through PTR_TO_BTF_ID, e.g. for expressions like p->arr[i].field. reg_bounds_sync() uses the description in two ways: - __reg_bound_offset() clears the low bits of var_off implied by the number of trailing zeros shared by base and step. - __update_bounds_from_step() intersects the 64-bit range with the line via cnum64_intersect_linear(), snapping smin/smax to actual points on it. This is the last step of the sync, as refining the range from var_off can move the bounds off the line. range_within() is extended to check whether the cur register's equation describes a subset of the points described by the old register's equation. Scalar ALU operations update base/step as follows: - ADD of a constant (including pointer + scalar): keep step, shift base; - MUL by a constant: scale step and base by the constant; - LSH by a constant: scale step and base by the corresponding power of two; - sign extension: keep the register unchanged if all values fit the target type, reset to base=0, step=1 otherwise; - All other operations reset to base=0, step=1. Branch (eq/neq) checks do not take into account base/step yet. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-19-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 7 ++ kernel/bpf/log.c | 2 + kernel/bpf/states.c | 26 ++++- kernel/bpf/verifier.c | 211 ++++++++++++++++++++++++++++++++++- 4 files changed, 240 insertions(+), 6 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b4396867c6f3..2560754ed56c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -139,6 +139,13 @@ struct bpf_reg_state { */ struct cnum64 r64; /* 64-bit range as circular number */ struct cnum32 r32; /* 32-bit range as circular number */ + /* + * The value described by this register, interpreted as s64, lies on + * a line described by a linear equation base + step * k. + * Invariant: base < step. + */ + u16 base; + u16 step; /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 87d137734db1..8ee7d7b71147 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -693,6 +693,8 @@ static void print_reg_state(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose_a("var_off=%s", tn_buf); } + if (reg->base != 0 || reg->step != 1) + verbose_a("step=%d+%d", reg->base, reg->step); } verbose(env, ")"); } diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 345079cf64d3..981328f5eb60 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -302,8 +302,30 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return cnum64_is_subset(old->r64, cur->r64) && - cnum32_is_subset(old->r32, cur->r32); + if (!cnum64_is_subset(old->r64, cur->r64) || + !cnum32_is_subset(old->r32, cur->r32)) + return false; + + if (old->step <= 1) + return true; + + if (cnum64_is_const(cur->r64)) + return imod((s64)cnum64_smin(cur->r64), old->step) == old->base; + + /* + * Both `old` and `cur` define some sets of points. + * Return true, if points defined by `cur` are a subset of points defined by `old`: + * - bounds for `cur` should be within bounds for `old`; + * - cur->step should be dividable by old->step; + * - cur->base should start at integer number of old->step + * steps from old->base. + * + * E.g. the following ranges are compatible: + * - old [0, 10] step 2 + * - new [2, 10] step 4 + */ + return cur->step % old->step == 0 && + (cur->base - old->base) % old->step == 0; } /* If in the old state two registers had the same id, then they need to have diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 47a632a6749a..f5b2500335a7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include #include @@ -1922,12 +1923,124 @@ static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, } } +static void reg_step_reset(struct bpf_reg_state *reg) +{ + reg->base = 0; + reg->step = 1; +} + +static void scalar_step_add(struct bpf_reg_state *dst_reg, + const struct bpf_reg_state *a, + const struct bpf_reg_state *b) +{ + const struct bpf_reg_state *reg; + s64 amount, tmp; + + if (tnum_is_const(b->var_off)) { + reg = a; + amount = (s64)b->var_off.value; + } else if (tnum_is_const(a->var_off)) { + reg = b; + amount = (s64)a->var_off.value; + } else { + reg_step_reset(dst_reg); + return; + } + + /* + * Let x be a possible signed value of the register, + * d = amount, s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * If the addition causes signed underflow, the resulting value + * is x + d + 2**64. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1) + * Keeping reg->step = s and updating reg->base to (b + d) mod s + * requires (1) to equal (b + d) mod s, which holds if and only + * if 2**64 mod s = 0. + * Signed overflow subtracts 2**64 and has the same requirement. + * + * Conservatively reset the stride if either wrap is possible. + */ + if (check_add_overflow(reg_smin(reg), amount, &tmp) || + check_add_overflow(reg_smax(reg), amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + + dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step; + dst_reg->step = reg->step; +} + +static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount) +{ + u16 step; + s64 tmp; + + /* + * Using the same notation as scalar_step_add(), with d > 0, + * the scaled value satisfies (x * d) mod (s * d) = b * d. + * If the multiplication wraps, its signed result can be represented + * as x * d - n * 2**64 for some nonzero integer n. + * + * Taking the result modulo s * d gives: + * (x * d - n * 2**64) mod (s * d) = + * (b * d - (n * 2**64 mod (s * d))) mod (s * d) (1) + * Updating dst_reg->base to b * d and dst_reg->step to s * d + * requires (1) to equal b * d, which holds if and only if + * n * 2**64 mod (s * d) = 0. + * + * Conservatively reset the stride if a wrap is possible. + */ + if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) || + check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) || + check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + /* + * The above established that `step = dst_reg->step * amount` + * does not overflow, therefore the base < step invariant holds + * after assignments below. + */ + dst_reg->base = dst_reg->base * amount; + dst_reg->step = step; +} + +/* + * Let x be a possible signed value of the register, + * s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * Truncating to N bits and then zero- or sign-extending x + * is equivalent to computing x + k * 2**N for some integer k. + * For example, sign-extending the low 8 bits of x = 255 can + * be expressed as 255 - 2**8 = -1. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s + * Keeping reg->base = b and reg->step = s requires this to equal b, + * which holds if and only if k * 2**N mod s = 0. + * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0, + * but (-1) mod 3 = 2, so the original base and step no longer hold. + * + * Conservatively reset the stride unless all possible values fit in + * the range where the conversion leaves them unchanged. + * Call before updating the register's bounds. Requires 0 < bits < 64. + */ +static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits) +{ + if (reg_umax(reg) >= (1ULL << bits)) + reg_step_reset(reg); +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); reg->r64 = cnum64_from_urange(imm, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); + reg_step_reset(reg); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -2185,8 +2298,24 @@ static void __reg_bound_offset(struct bpf_reg_state *reg) struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), tnum_range(reg_u32_min(reg), reg_u32_max(reg))); + u32 trailing_zero_bits; + u16 base = reg->base; + u16 step = reg->step; reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); + + if (base == 0) + trailing_zero_bits = count_trailing_zeros(step); + else + trailing_zero_bits = min(count_trailing_zeros(base), + count_trailing_zeros(step)); + reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits)); +} + +static void __update_bounds_from_step(struct bpf_reg_state *reg) +{ + reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step); + deduce_bounds_32_from_64(reg); } static bool range_bounds_violation(struct bpf_reg_state *reg); @@ -2208,6 +2337,15 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) * then new var_off is (0; 0x7f...fc) which improves our umax. */ __update_reg_bounds(reg); + /* + * Both tnum and base+step define a sequence of points, + * potentially w/o intersection, or intersecting after N + * tnum_step()'s, where N can be large. On the other hand, + * step+base *must* be in sync with r64 in order for + * array traversal logic starting at reg_umin() to be sound. + * Hence, place this as a final step in the reg_bounds_sync(). + */ + __update_bounds_from_step(reg); } static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) @@ -2251,17 +2389,24 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, goto out; } + if (reg->step == 0 || reg->base >= reg->step) { + msg = "invalid base/step"; + goto out; + } + return 0; out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " - "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", + "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u", ctx, msg, reg->r64.base, reg->r64.size, reg->r32.base, reg->r32.size, - reg->var_off.value, reg->var_off.mask); + reg->var_off.value, reg->var_off.mask, + reg->base, reg->step); if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); + reg_step_reset(reg); return 0; } @@ -2272,6 +2417,7 @@ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; __mark_reg_unbounded(reg); + reg_step_reset(reg); } /* Mark a register as having a completely unknown (scalar) value, @@ -5902,6 +6048,7 @@ static int check_buffer_access(struct bpf_verifier_env *env, /* BPF architecture zero extends alu32 ops into 64-bit registesr */ static void zext_32_to_64(struct bpf_reg_state *reg) { + reg_step_check_unsigned(reg, 32); reg->var_off = tnum_subreg(reg->var_off); reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); } @@ -5911,13 +6058,14 @@ static void zext_32_to_64(struct bpf_reg_state *reg) */ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) { - u64 mask; + u64 mask = (1ULL << (size * 8)) - 1; + + reg_step_check_unsigned(reg, size * 8); /* clear high bits in bit representation */ reg->var_off = tnum_cast(reg->var_off, size); /* fix arithmetic bounds */ - mask = ((u64)1 << (size * 8)) - 1; if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); else @@ -5954,6 +6102,14 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; u64 top_smax_value, top_smin_value; u64 num_bits = size * 8; + s64 limit = 1LL << (num_bits - 1); + + /* If every value fits the signed N-bit range sign extension is a noop. */ + if (reg_smin(reg) >= -limit && reg_smax(reg) < limit) + return; + + /* See commen at reg_step_check_unsigned(). */ + reg_step_reset(reg); if (tnum_is_const(reg->var_off)) { u64_cval = reg->var_off.value; @@ -6020,6 +6176,16 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) u32 top_smax_value, top_smin_value; u32 num_bits = size * 8; + /* + * If value fits the non-negative half of the signed N-bit range, + * sign extension is a noop. + */ + if (reg_umax(reg) < (1ULL << (num_bits - 1))) + return; + + /* See comment at reg_step_check_unsigned(). */ + reg_step_reset(reg); + if (tnum_is_const(reg->var_off)) { u32_val = reg->var_off.value; if (size == 1) @@ -6729,6 +6895,7 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, val); + scalar_step_add(dst_reg, dst_reg, &fake_reg); scalar32_min_max_add(dst_reg, &fake_reg); scalar_min_max_add(dst_reg, &fake_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off); @@ -15693,6 +15860,24 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return 0; } +static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + if (tnum_is_const(src_reg->var_off)) + scalar_step_scale(dst_reg, src_reg->var_off.value); + else + reg_step_reset(dst_reg); +} + +static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + u64 amount = src_reg->var_off.value; + + if (tnum_is_const(src_reg->var_off) && amount < 64) + scalar_step_scale(dst_reg, 1ULL << amount); + else + reg_step_reset(dst_reg); +} + /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a @@ -15861,6 +16046,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ + scalar_step_add(dst_reg, ptr_reg, off_reg); dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -15922,6 +16108,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn if ((!known && smin_val < 0) || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } + reg_step_reset(dst_reg); break; case BPF_AND: case BPF_OR: @@ -16629,6 +16816,7 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins * Bounds will be re-derived from the new tnum later. */ __mark_reg_unbounded(dst_reg); + reg_step_reset(dst_reg); } /* For bswap16/32, truncate dst register to match the swapped size */ if (insn->imm == 16 || insn->imm == 32) @@ -16755,6 +16943,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, */ switch (opcode) { case BPF_ADD: + scalar_step_add(dst_reg, dst_reg, &src_reg); scalar32_min_max_add(dst_reg, &src_reg); scalar_min_max_add(dst_reg, &src_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); @@ -16763,6 +16952,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &src_reg); scalar_min_max_sub(dst_reg, &src_reg); dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); + reg_step_reset(dst_reg); break; case BPF_NEG: env->fake_reg[0] = *dst_reg; @@ -16770,8 +16960,10 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); scalar_min_max_sub(dst_reg, &env->fake_reg[0]); dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); + reg_step_reset(dst_reg); break; case BPF_MUL: + scalar_step_mul(dst_reg, &src_reg); dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); scalar32_min_max_mul(dst_reg, &src_reg); scalar_min_max_mul(dst_reg, &src_reg); @@ -16792,6 +16984,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_sdiv(dst_reg, &src_reg); else scalar_min_max_udiv(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ @@ -16807,6 +17000,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_smod(dst_reg, &src_reg); else scalar_min_max_umod(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { @@ -16817,6 +17011,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { @@ -16827,13 +17022,16 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_XOR: dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); scalar32_min_max_xor(dst_reg, &src_reg); scalar_min_max_xor(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_LSH: + scalar_step_lsh(dst_reg, &src_reg); if (alu32) scalar32_min_max_lsh(dst_reg, &src_reg); else @@ -16844,15 +17042,18 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_rsh(dst_reg, &src_reg); else scalar_min_max_rsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_ARSH: if (alu32) scalar32_min_max_arsh(dst_reg, &src_reg); else scalar_min_max_arsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_END: scalar_byte_swap(dst_reg, insn); + reg_step_reset(dst_reg); break; default: break; @@ -17759,6 +17960,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * violations if we're on a dead branch. */ __mark_reg_unbounded(reg1); + reg_step_reset(reg1); if (is_jmp32) { t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); @@ -18081,6 +18283,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s reg->delta = saved_off; reg->id = saved_id; + scalar_step_add(reg, reg, &fake_reg); scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); From 89e72f81a45576b0abe31a509a2660f3dc0c5d21 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:38 -0700 Subject: [PATCH 408/660] bpf: Varying offset access support for PTR_TO_BTF_ID pointers Relax the requirements for reads like `p->arr[i].field`, where `p` is a PTR_TO_BTF_ID, by allowing `i` to be a non-constant value. This is useful for loops over arrays accessed from a PTR_TO_BTF_ID when the verifier widens the loop induction variable. Check that the access stays within array bounds based on the access register's min and max bounds. Check that the access is aligned with array elements using the register's `base` and `step`. Technically, modify btf_struct_access() such that: - it identifies `field` by walking BTF from the minimum offset (the register's reg_smin + static instruction offset); - it remembers which arrays contain `field`; - it checks that offsets `k * step` stay within the array and point to the same `field`; - it does not check upper bounds for flexible arrays (as is already the case for constant-offset access). Support for flexible arrays nested inside trailing struct members (e.g. p->nested.data[i], with data declared as int data[]) is left for future work. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-20-bcba9bc53ac9@gmail.com --- kernel/bpf/btf.c | 161 +++++++++++++++++++++++++++++++++++------- kernel/bpf/states.c | 1 + kernel/bpf/verifier.c | 38 ++++++---- 3 files changed, 160 insertions(+), 40 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 0630675377aa..f4d119f3d5ef 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2079,7 +2079,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_id: id of u32 * *total_nelems: (x * y). Hence, individual elem size is * (*type_size / *total_nelems) - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. * * type: is not an array (e.g. const struct X) * return type: type "struct X" @@ -2087,7 +2088,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_type: same as return type ("struct X") * *elem_id: 0 * *total_nelems: 1 - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. */ static const struct btf_type * __btf_resolve_size(const struct btf *btf, const struct btf_type *type, @@ -2120,7 +2122,8 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_TYPE_TAG: - id = type->type; + if (!array_type) + id = type->type; type = btf_type_by_id(btf, type->type); break; @@ -7549,6 +7552,14 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } EXPORT_SYMBOL_GPL(btf_ctx_access); +#define MAX_ARRAYS_WALK 16 + +/* Description of an array crossed while walking a BTF access chain */ +struct array_access { + u32 id; /* BTF_KIND_ARRAY type id */ + u32 off; /* offset of the access within the array */ +}; + enum bpf_struct_walk_result { /* < 0 error */ WALK_SCALAR = 0, @@ -7557,10 +7568,43 @@ enum bpf_struct_walk_result { WALK_STRUCT, }; +static const struct btf_member *find_flex_member(const struct btf *btf, const struct btf_type *t) +{ + const struct btf_array *array; + const struct btf_type *mtype; + const struct btf_member *member; + u32 vlen = btf_type_vlen(t); + + if (vlen == 0) + return NULL; + + member = btf_type_member(t) + vlen - 1; + mtype = btf_type_skip_modifiers(btf, member->type, NULL); + if (!btf_type_is_array(mtype)) + return NULL; + + array = (const struct btf_array *)(mtype + 1); + if (array->nelems != 0) + return NULL; + + return member; +} + +static int push_array(struct array_access *arrays, u32 *arrays_cnt, struct array_access item) +{ + if (arrays == NULL) + return 0; + if (*arrays_cnt == MAX_ARRAYS_WALK) + return -1; + arrays[(*arrays_cnt)++] = item; + return 0; +} + static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name, bool walk_flex_arrays) + const char **field_name, bool walk_flex_arrays, + struct array_access *arrays, u32 *arrays_cnt) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7595,23 +7639,20 @@ again: /* If the last element is a variable size array, we may * need to relax the rule. */ - if (vlen == 0) - goto error; - - member = btf_type_member(t) + vlen - 1; - mtype = btf_type_skip_modifiers(btf, member->type, - NULL); - if (!btf_type_is_array(mtype)) - goto error; - - array_elem = (struct btf_array *)(mtype + 1); - if (array_elem->nelems != 0) + member = find_flex_member(btf, t); + if (!member) goto error; moff = __btf_member_bit_offset(t, member) / 8; if (off < moff) goto error; + mtype = btf_type_skip_modifiers(btf, member->type, &mid); + array_elem = (struct btf_array *)(mtype + 1); + + if (push_array(arrays, arrays_cnt, (struct array_access){ mid, off - moff })) + return -E2BIG; + /* allow structure and integer */ t = btf_type_skip_modifiers(btf, array_elem->type, NULL); @@ -7733,6 +7774,9 @@ error: * the array's element as long as it is * within the mtrue_end boundary. */ + if (push_array(arrays, arrays_cnt, + (struct array_access){ mid, off - moff })) + return -E2BIG; /* skip empty array */ if (moff == mtrue_end) @@ -7825,15 +7869,29 @@ error: int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, - int off, int size, enum bpf_access_type atype __maybe_unused, + int _off, int size, enum bpf_access_type atype __maybe_unused, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name) { const struct btf *btf = reg->btf; enum bpf_type_flag tmp_flag = 0; + struct array_access arrays[MAX_ARRAYS_WALK]; + const struct btf_member *member; const struct btf_type *t; + u32 i, arrays_cnt = 0; u32 id = reg->btf_id; - int err; + s64 min_off, max_off, flex_off; + int err, off, ret; + + if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) || + check_add_overflow(reg_smax(reg), (s64)_off, &max_off) || + min_off != (int)min_off) { + bpf_log(log, + "offset computation overflows: register offset range is [%lld, %lld], instruction offset is %d\n", + reg_smin(reg), reg_smax(reg), _off); + return -EINVAL; + } + off = min_off; while (type_is_alloc(reg->type)) { struct btf_struct_meta *meta; @@ -7860,26 +7918,30 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, - field_name, !type_is_alloc(reg->type)); - + field_name, !type_is_alloc(reg->type), arrays, &arrays_cnt); switch (err) { case WALK_PTR: /* For local types, the destination register cannot * become a pointer again. */ - if (type_is_alloc(reg->type)) - return SCALAR_VALUE; + if (type_is_alloc(reg->type)) { + ret = SCALAR_VALUE; + goto check_variable_offset; + } /* If we found the pointer or scalar on t+off, * we're done. */ *next_btf_id = id; *flag = tmp_flag; - return PTR_TO_BTF_ID; + ret = PTR_TO_BTF_ID; + goto check_variable_offset; case WALK_PTR_UNTRUSTED: *flag = MEM_RDONLY | PTR_UNTRUSTED; - return PTR_TO_MEM; + ret = PTR_TO_MEM; + goto check_variable_offset; case WALK_SCALAR: - return SCALAR_VALUE; + ret = SCALAR_VALUE; + goto check_variable_offset; case WALK_STRUCT: /* We found nested struct, so continue the search * by diving in it. At this point the offset is @@ -7899,6 +7961,55 @@ int btf_struct_access(struct bpf_verifier_log *log, } while (t); return -EINVAL; + +check_variable_offset: + if (min_off == max_off) + return ret; + + /* Find an offset at which access would go to a flexible array tail (if any). */ + t = btf_type_skip_modifiers(btf, reg->btf_id, NULL); + member = find_flex_member(btf, t); + flex_off = member ? __btf_member_bit_offset(t, member) / 8 : S64_MAX; + + /* + * If this is a varying offset access, the step recorded within a register + * should correspond to one of the arrays visited while walking. + */ + for (i = 0; i < arrays_cnt; i++) { + s64 array_start, array_end, access_end; + u32 asize, esize, elem_id; + + /* + * 'arrays' records top-level arrays only, use __btf_resolve_size() + * to get flattened representation. + */ + t = btf_type_by_id(btf, arrays[i].id); + if (IS_ERR(__btf_resolve_size(btf, t, &asize, NULL, &elem_id, NULL, NULL)) || + IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, elem_id), &esize))) + continue; + /* Make sure every accessed offset lands on the same field of some element. */ + if (esize == 0 || reg->step % esize != 0) + continue; + + array_start = min_off - arrays[i].off; + /* An array within the flexible tail has no upper bound to exceed */ + if (array_start >= flex_off && btf_type_is_array(t) && + btf_type_array(t)->nelems == 0) + return ret; + + /* the 'size' bytes read at max_off must stay within the array */ + if (check_add_overflow(array_start, (s64)asize, &array_end) || + check_add_overflow(max_off, (s64)size, &access_end) || + access_end > array_end) + continue; + return ret; + } + + bpf_log(log, + "invalid variable offset access into struct %s: offsets [%lld, %lld] size %d step %d do not match any array\n", + btf_name_by_offset(btf, btf_type_by_id(btf, reg->btf_id)->name_off), + min_off, max_off, size, reg->step); + return -EINVAL; } /* Check that two BTF types, each specified as an BTF object + id, are exactly @@ -7939,7 +8050,7 @@ again: if (!type) return false; err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, - walk_flex_arrays); + walk_flex_arrays, NULL, NULL); if (err != WALK_STRUCT) return false; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 981328f5eb60..2f6a7620eb72 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -633,6 +633,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_BUF: case PTR_TO_TP_BUFFER: case PTR_TO_STACK: + case PTR_TO_BTF_ID: /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f5b2500335a7..192efb722997 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6554,6 +6554,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, const char *field_name = NULL; enum bpf_type_flag flag = 0; u32 btf_id = 0; + s64 min_off; int ret; if (!env->allow_ptr_leaks) { @@ -6569,36 +6570,31 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EINVAL; } - if (!tnum_is_const(reg->var_off)) { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + if (check_add_overflow(reg_smin(reg), off, &min_off)) { verbose(env, - "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", - reg_arg_name(env, argno), tname, off, tn_buf); + "%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n", + reg_arg_name(env, argno), tname, reg_smin(reg), off); return -EACCES; } - off += reg->var_off.value; - - if (off < 0) { + if (min_off < 0) { verbose(env, - "%s is ptr_%s invalid negative access: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s invalid negative access: off=%lld\n", + reg_arg_name(env, argno), tname, min_off); return -EACCES; } if (reg->type & MEM_USER) { verbose(env, - "%s is ptr_%s access user memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access user memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } if (reg->type & MEM_PERCPU) { verbose(env, - "%s is ptr_%s access percpu memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access percpu memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } @@ -6608,6 +6604,18 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, + "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), tname, off, tn_buf); + return -EACCES; + } + + off += reg->var_off.value; + if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); return -EFAULT; From f414d20879ae1bac413619b8352701a7da2a1c0e Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:39 -0700 Subject: [PATCH 409/660] bpf: Save DFS postorder numbers for program instructions Dominator intersections and SCEV worklist scheduling need to compare instructions by their DFS postorder rank. Save per-instruction postorder numbers alongside the existing postorder sequence. This commit modifies bpf_compute_postorder() to actually do the DFS traversal, instead of scheduling traversal of all siblings at once. Consider a graph: A -> B A -> C C -> B Old result: C, B, A DFS result: B, C, A Fixes: efcda22aa541 ("bpf: compute instructions postorder per subprogram") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-21-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/cfg.c | 56 ++++++++++++++++++++++-------------- kernel/bpf/const_fold.c | 2 ++ kernel/bpf/verifier.c | 1 + 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 2560754ed56c..bd7733fbe930 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1036,6 +1036,7 @@ struct bpf_verifier_env { * see bpf_subprog_info->postorder_start. */ int *insn_postorder; + int *postorder_nums; int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index d8a579680e5b..bce7b5abcfda 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -767,6 +767,11 @@ err_free: return ret; } +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + /* * For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range * [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start) @@ -774,43 +779,52 @@ err_free: */ int bpf_compute_postorder(struct bpf_verifier_env *env) { - u32 cur_postorder, i, top, stack_sz, s; - int *stack = NULL, *postorder = NULL, *state = NULL; - struct bpf_iarray *succ; + int *stack = NULL, *postorder = NULL, *postorder_nums = NULL; + int subprog_idx, stack_sz, cur, s, cur_postorder, start; + struct dfs_state *state = NULL; + struct bpf_iarray *succ = NULL; + postorder_nums = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - if (!postorder || !state || !stack) { + state = kvzalloc_objs(struct dfs_state, env->prog->len, GFP_KERNEL_ACCOUNT); + if (!postorder_nums || !postorder || !stack || !state) { + kvfree(postorder_nums); kvfree(postorder); - kvfree(state); kvfree(stack); + kvfree(state); return -ENOMEM; } cur_postorder = 0; - for (i = 0; i < env->subprog_cnt; i++) { - env->subprog_info[i].postorder_start = cur_postorder; - stack[0] = env->subprog_info[i].start; + for (subprog_idx = 0; subprog_idx < env->subprog_cnt; subprog_idx++) { + start = env->subprog_info[subprog_idx].start; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + stack[0] = start; stack_sz = 1; + state[start].traversed = true; do { - top = stack[stack_sz - 1]; - state[top] |= DISCOVERED; - if (state[top] & EXPLORED) { - postorder[cur_postorder++] = top; + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + postorder_nums[cur] = cur_postorder; + postorder[cur_postorder] = cur; + cur_postorder++; stack_sz--; continue; } - succ = bpf_insn_successors(env, top); - for (s = 0; s < succ->cnt; ++s) { - if (!state[succ->items[s]]) { - stack[stack_sz++] = succ->items[s]; - state[succ->items[s]] |= DISCOVERED; - } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + stack_sz++; + continue; } - state[top] |= EXPLORED; + state[cur].next_succ++; } while (stack_sz); } - env->subprog_info[i].postorder_start = cur_postorder; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + env->cfg.postorder_nums = postorder_nums; env->cfg.insn_postorder = postorder; env->cfg.cur_postorder = cur_postorder; kvfree(stack); diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b1528adbeb79..8dd8c5958eb8 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -398,5 +398,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) /* recompute postorder, since CFG has changed */ kvfree(env->cfg.insn_postorder); env->cfg.insn_postorder = NULL; + kvfree(env->cfg.postorder_nums); + env->cfg.postorder_nums = NULL; return bpf_compute_postorder(env); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 192efb722997..91f114947b8b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23221,6 +23221,7 @@ err_free_env: vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env); + kvfree(env->cfg.postorder_nums); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); From ceea9085f2d131987903d7965d3738168275bf4a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:40 -0700 Subject: [PATCH 410/660] bpf: Move the live-register and SCC printout to a standalone function Tests for multiple analyses performed by the verifier need the verifier log to contain analysis results alongside the program disassembly. This patch moves the log-level-2 program dump from bpf_compute_live_registers() to a standalone function called from bpf_check(), in order to provide a common logging function for such analyses (and thus avoid printing program disassembly multiple times). Rename the dump header from "Live regs before insn:" to "Program dump (scc? insn#: live_regs_before):" and print available source-line information before the corresponding instructions. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-22-bcba9bc53ac9@gmail.com --- kernel/bpf/liveness.c | 28 +------------- kernel/bpf/verifier.c | 37 +++++++++++++++++++ .../selftests/bpf/progs/verifier_live_stack.c | 2 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 8d8428ccc505..e657e6d95408 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2664,8 +2664,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; - u64 pos, insn_pos; - int err = 0, i, j, subprog, start, end; + int err = 0, i, subprog, start, end; bool changed, ret_reg_pair; /* Use the following algorithm: @@ -2754,31 +2753,6 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); } - if (env->log.level & BPF_LOG_LEVEL2) { - verbose(env, "Live regs before insn:\n"); - for (i = 0; i < insn_cnt; ++i) { - if (env->insn_aux_data[i].scc) - verbose(env, "%3d ", env->insn_aux_data[i].scc); - else - verbose(env, " "); - verbose(env, "%3d: ", i); - for (j = BPF_REG_0; j < BPF_REG_10; ++j) - if (insn_aux[i].live_regs_before & BIT(j)) - verbose(env, "%d", j); - else - verbose(env, "."); - verbose(env, " "); - pos = env->log.end_pos; - bpf_verbose_insn(env, &insns[i]); - insn_pos = env->log.end_pos; - if (insn_aux[i].zext_dst) - verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); - verbose(env, "\n"); - if (bpf_is_ldimm64(&insns[i])) - i++; - } - } - out: kvfree(state); return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 91f114947b8b..9260ee290072 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22863,6 +22863,40 @@ static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) return false; } +/* Various log level 2 information about the program */ +static void log_program(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; + struct bpf_insn *insns = env->prog->insnsi; + u32 insn_cnt = env->prog->len; + u64 pos, insn_pos; + u32 i, j; + + verbose(env, "Program dump (scc? insn#: live_regs_before):\n"); + for (i = 0; i < insn_cnt; ++i) { + verbose_linfo(env, i, " ; "); + if (env->insn_aux_data[i].scc) + verbose(env, "%3d ", env->insn_aux_data[i].scc); + else + verbose(env, " "); + verbose(env, "%3d: ", i); + for (j = BPF_REG_0; j < BPF_REG_10; ++j) + if (insn_aux[i].live_regs_before & BIT(j)) + verbose(env, "%d", j); + else + verbose(env, "."); + verbose(env, " "); + pos = env->log.end_pos; + bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); + if (bpf_is_ldimm64(&insns[i])) + i++; + } +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -23073,6 +23107,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + if (env->log.level & BPF_LOG_LEVEL2) + log_program(env); + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 3abe08cc6595..6e8f50618580 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -66,7 +66,7 @@ __msg("stack use/def subprog#0 must_write_not_same_slot (d0,cs0):") * but the write conservatively marks the whole frame as may_def. */ __msg("6: (7b) *(u64 *)(r2 +0) = r0 ; may_def: fp0-8..-{{(512|2048)}}") -__msg("Live regs before insn:") +__msg("Program dump") __naked void must_write_not_same_slot(void) { asm volatile ( From 6254fb69328963be95ff9b2760e879a9220ae6c3 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:41 -0700 Subject: [PATCH 411/660] bpf: Compute immediate dominators SCEV needs to find an exit condition that dominates a loop's backedge; such conditions bound every loop iteration. Compute the immediate dominator tree as a prerequisite for identifying such conditions. Use the iterative reverse-postorder algorithm from: "A Simple, Fast Dominance Algorithm" by Cooper et al. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-23-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 8 ++ kernel/bpf/Makefile | 2 +- kernel/bpf/loops.c | 158 +++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 8 +- 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 kernel/bpf/loops.c diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bd7733fbe930..59c0670af0e9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -650,6 +650,11 @@ struct bpf_iarray { u32 items[]; }; +#define iarray_for_each(item, arr) \ + for (int ___idx = 0; \ + ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ + ___idx++) + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -1112,6 +1117,7 @@ struct bpf_verifier_env { u32 scc_cnt; struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; + int *idoms; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1888,4 +1894,6 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); int bpf_flip_opcode(u32 opcode); u8 bpf_rev_opcode(u8 opcode); +int bpf_compute_idoms(struct bpf_verifier_env *env); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index c1f9b0d3468d..6210601eb40b 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c new file mode 100644 index 000000000000..0db823ee42f7 --- /dev/null +++ b/kernel/bpf/loops.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include + +static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) +{ + struct bpf_iarray *succ, *preds, **result; + struct bpf_prog *prog = env->prog; + u32 *num_preds, i, s, len = prog->len; + struct bpf_insn *insn; + size_t sz; + void *tmp; + + num_preds = kvcalloc(prog->len, sizeof(u32), GFP_KERNEL_ACCOUNT); + if (!num_preds) + return NULL; + + /* + * 'result' layout: + * - array of pointers (struct bpf_iarray *)[len] + * - struct bpf_iarray one after another + */ + sz = size_mul(len, sizeof(*result) + sizeof(**result)); + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + sz = size_add(sz, array_size(succ->cnt, sizeof(*succ->items))); + iarray_for_each(s, succ) { + num_preds[s]++; + } + if (bpf_is_ldimm64(insn)) + i++; + } + + result = kvzalloc(sz, GFP_KERNEL_ACCOUNT); + if (!result) { + kvfree(num_preds); + return NULL; + } + + tmp = (void *)&result[len]; + for (i = 0; i < len; i++) { + result[i] = tmp; + tmp += struct_size(result[i], items, num_preds[i]); + } + + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + preds = result[s]; + preds->items[preds->cnt++] = i; + } + if (bpf_is_ldimm64(insn)) + i++; + } + + kvfree(num_preds); + return result; +} + +static int idoms_intersect(struct bpf_verifier_env *env, int a, int b) +{ + int *postorder_nums = env->cfg.postorder_nums; + int *idoms = env->idoms; + + while (a != b) { + while (postorder_nums[a] < postorder_nums[b]) { + a = idoms[a]; + } + while (postorder_nums[b] < postorder_nums[a]) { + b = idoms[b]; + } + } + return a; +} + +/* See "A Simple, Fast Dominance Algorithm" by Cooper et al. for details. */ +static int compute_subprog_idoms(struct bpf_verifier_env *env, struct bpf_iarray **preds, + int subprog_idx) +{ + struct bpf_subprog_info *subprog = &env->subprog_info[subprog_idx]; + int start = subprog->start; + int po_first = subprog->postorder_start; + int po_last = (subprog + 1)->postorder_start - 1; + int *idoms = env->idoms; + int po_num, pred; + u32 work = 0; + bool changed; + + idoms[start] = start; + changed = true; + do { + changed = false; + /* + * Iterate in reverse postorder, skipping the subprog entry: + * insn_postorder[po_last] is the subprogram entry. + */ + for (po_num = po_last - 1; po_num >= po_first; po_num--) { + int idx = env->cfg.insn_postorder[po_num]; + int new_idom = -1; + + iarray_for_each(pred, preds[idx]) { + if (++work % 1024 == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + if (idoms[pred] == -1) + continue; + if (new_idom == -1) + new_idom = pred; + else + new_idom = idoms_intersect(env, pred, new_idom); + } + if (new_idom != -1 && idoms[idx] != new_idom) { + idoms[idx] = new_idom; + changed = true; + } + } + } while (changed); + idoms[start] = -1; + return 0; +} + +int bpf_compute_idoms(struct bpf_verifier_env *env) +{ + struct bpf_iarray **preds; + u32 len = env->prog->len; + int *idoms, i, err = 0; + + preds = compute_predecessors(env); + if (!preds) + return -ENOMEM; + + idoms = kvcalloc(len, sizeof(*idoms), GFP_KERNEL_ACCOUNT); + if (!idoms) { + kvfree(preds); + return -ENOMEM; + } + + env->idoms = idoms; + for (i = 0; i < len; i++) + idoms[i] = -1; + + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_subprog_idoms(env, preds, i); + if (err) + break; + } + + kvfree(preds); + return err; +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9260ee290072..5a29d05c523c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22872,13 +22872,14 @@ static void log_program(struct bpf_verifier_env *env) u64 pos, insn_pos; u32 i, j; - verbose(env, "Program dump (scc? insn#: live_regs_before):\n"); + verbose(env, "Program dump (scc? idom insn#: live_regs_before):\n"); for (i = 0; i < insn_cnt; ++i) { verbose_linfo(env, i, " ; "); if (env->insn_aux_data[i].scc) verbose(env, "%3d ", env->insn_aux_data[i].scc); else verbose(env, " "); + verbose(env, "%3d ", env->idoms[i]); verbose(env, "%3d: ", i); for (j = BPF_REG_0; j < BPF_REG_10; ++j) if (insn_aux[i].live_regs_before & BIT(j)) @@ -23103,6 +23104,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, prepare_subprog_args(env); + ret = bpf_compute_idoms(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; @@ -23266,6 +23271,7 @@ err_free_env: kvfree(env->callx_edges); kvfree(env->func_ptrs); bpf_diag_free(env); + kvfree(env->idoms); kvfree(env); return ret; } From 9ae7f5e8bf1d8be74797f937cdbeb730178a8c3d Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:42 -0700 Subject: [PATCH 412/660] bpf: Compute loop hierarchy SCEV needs to process loops from innermost to outermost, summarizing results for nested loops. Add a pass to compute the loop structure. Use a non-recursive adaptation of the algorithm from: "A New Algorithm for Identifying Loops in Decompilation" by Wei et al. Record the loop hierarchy per instruction: - The field insn_aux_data->loop_header records the innermost loop header containing it. - The field insn_aux_data->loop records additional information about the loop if this instruction is a loop header (exits, backedges, irreducibility). Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-24-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 46 ++++ kernel/bpf/fixups.c | 18 ++ kernel/bpf/loops.c | 439 +++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 12 +- 4 files changed, 514 insertions(+), 1 deletion(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 59c0670af0e9..d1ebbfef5455 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -655,6 +655,41 @@ struct bpf_iarray { ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ ___idx++) +#define MAX_BACKEDGES 16 +#define MAX_LOOP_EXITS 256 + +/* Number of exit conditions dominating a backedge. */ +#define MAX_LATCHES 4 + +struct bpf_backedge { + int from; + /* + * Conditional jumps within the loop that dominate 'from' and have + * a successor outside the loop, nearest first. latches[0] is the + * latch; the remaining entries are additional exit conditions. + * Each usable condition gives an upper bound on the iteration count, + * so keeping only a subset is sound. + */ + int latches[MAX_LATCHES]; + u8 latches_cnt; +}; + +struct bpf_loop_exit { + int from; /* instruction inside the loop */ + int to; /* instruction outside the loop */ +}; + +struct bpf_loop { + struct bpf_backedge backedges[MAX_BACKEDGES]; + /* edges exiting from this loop, includes edges from nested loops */ + struct bpf_loop_exit *exits; + int backedges_cnt; + int exits_cnt; + bool irreducible; + bool backedges_overflow; + bool exits_overflow; +}; + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -752,6 +787,13 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* + * Header index of the innermost loop containing this instruction, -1 if none. + * For a loop header, identifies the parent loop instead. + */ + s32 loop_header; + /* additional information about the loop if this instruction is a loop header */ + struct bpf_loop *loop; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ @@ -1879,6 +1921,7 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct bpf_attach_target_info *tgt_info); /* Functions in fixups.c, called from bpf_check() */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); @@ -1895,5 +1938,8 @@ int bpf_flip_opcode(u32 opcode); u8 bpf_rev_opcode(u8 opcode); int bpf_compute_idoms(struct bpf_verifier_env *env); +int bpf_compute_loops(struct bpf_verifier_env *env); +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 206cc9a614a9..b2fa7e8bedf3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -590,6 +590,22 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, return 0; } +/* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + int end = start + len; + int i; + + for (i = start; i < end; i++) { + if (aux_data[i].loop) { + kvfree(aux_data[i].loop->exits); + kvfree(aux_data[i].loop); + aux_data[i].loop = NULL; + } + } +} + static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; @@ -602,6 +618,8 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); + bpf_clear_insn_aux_data(env, off, cnt); + err = bpf_remove_insns(env->prog, off, cnt); if (err) return err; diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c index 0db823ee42f7..8a6dc805e559 100644 --- a/kernel/bpf/loops.c +++ b/kernel/bpf/loops.c @@ -156,3 +156,442 @@ int bpf_compute_idoms(struct bpf_verifier_env *env) kvfree(preds); return err; } + +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + +struct loops_dfs { + struct dfs_state *state; + int *dfs_pos; + int *stack; +}; + +static void mark_irreducible(struct bpf_verifier_env *env, int h) +{ + env->insn_aux_data[h].loop->irreducible = true; +} + +static void add_backedge(struct bpf_verifier_env *env, int from, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[h].loop; + int cnt = loop->backedges_cnt; + + if (cnt == MAX_BACKEDGES) { + loop->backedges_overflow = true; + return; + } + loop->backedges[cnt].from = from; + loop->backedges[cnt].latches_cnt = 0; + loop->backedges_cnt++; +} + +static int add_exit(struct bpf_loop *loop, int from, int to) +{ + if (loop->exits_overflow) + return 0; + if (loop->exits_cnt == MAX_LOOP_EXITS) { + loop->exits_overflow = true; + return 0; + } + if (!loop->exits) { + loop->exits = kvcalloc(MAX_LOOP_EXITS, sizeof(*loop->exits), GFP_KERNEL_ACCOUNT); + if (!loop->exits) + return -ENOMEM; + } + loop->exits[loop->exits_cnt] = (struct bpf_loop_exit) { + .from = from, + .to = to, + }; + loop->exits_cnt++; + return 0; +} + +static int mark_as_header(struct bpf_verifier_env *env, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + if (!aux[h].loop) { + aux[h].loop = kvzalloc_obj(struct bpf_loop, GFP_KERNEL_ACCOUNT); + if (!aux[h].loop) + return -ENOMEM; + } + return 0; +} + +static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, int n, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int *dfs_pos = dfs->dfs_pos; + int err, nh; + + err = mark_as_header(env, h); + if (err) + return err; + + /* Don't encode self-loops, otherwise can't reflect loops nesting structure. */ + if (n == h) + return 0; + + /* Make sure that loop headers up the chain are sorted by dfs_pos. */ + while (aux[n].loop_header != -1) { + nh = aux[n].loop_header; + if (nh == h) + return 0; + if (dfs_pos[nh] < dfs_pos[h]) { + aux[n].loop_header = h; + n = h; + h = nh; + } else { + n = nh; + } + } + aux[n].loop_header = h; + return 0; +} + +static bool is_cond_jmp_insn(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + if (class != BPF_JMP && class != BPF_JMP32) + return false; + + switch (opcode) { + case BPF_JEQ: + case BPF_JGE: + case BPF_JGT: + case BPF_JLE: + case BPF_JLT: + case BPF_JNE: + case BPF_JSET: + case BPF_JSGE: + case BPF_JSGT: + case BPF_JSLE: + case BPF_JSLT: + return true; + default: + return false; + } +} + +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + return aux[idx].loop ? idx : aux[idx].loop_header; +} + +/* + * Collect conditional jumps within n's loop that dominate 'n' and have + * a successor outside the loop, nearest first. + */ +static void find_dominating_conditions(struct bpf_verifier_env *env, int n, + struct bpf_backedge *backedge) +{ + struct bpf_insn *insns = env->prog->insnsi; + bool t_exits_loop, f_exits_loop; + int n_loop, t_loop, f_loop; + int *idoms = env->idoms; + + n_loop = bpf_loop_at_index(env, n); + if (idoms_intersect(env, n, n_loop) != n_loop) + return; + while (n >= 0) { + if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) { + t_loop = bpf_loop_at_index(env, n + insns[n].off + 1); + f_loop = bpf_loop_at_index(env, n + 1); + t_exits_loop = t_loop != n_loop && !bpf_is_nested_loop(env, t_loop, n_loop); + f_exits_loop = f_loop != n_loop && !bpf_is_nested_loop(env, f_loop, n_loop); + if (t_exits_loop || f_exits_loop) { + if (backedge->latches_cnt == MAX_LATCHES) + return; + backedge->latches[backedge->latches_cnt++] = n; + } + } + if (n == n_loop) + break; + n = idoms[n]; + } +} + +/* + * As described in "A New Algorithm for Identifying Loops in Decompilation" by Wei et al, + * adapted to be non-recursive. + */ +static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_dfs *dfs, + int subprog_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct dfs_state *state = dfs->state; + int start = env->subprog_info[subprog_idx].start; + int *dfs_pos = dfs->dfs_pos; + int *stack = dfs->stack; + int s, h, err, cur, stack_sz; + struct bpf_iarray *succ; + u32 i; + + stack[0] = start; + state[start].traversed = true; + state[start].next_succ = 0; + dfs_pos[start] = 1; + stack_sz = 1; + i = 0; + do { + /* + * The algorithm should be very fast in practice, + * guard against pathological inputs, just in case. + */ + if ((++i % 1024) == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + dfs_pos[cur] = 0; + stack_sz--; + continue; + } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + /* Case A: start -> ... -> cur -> s [unexplored] */ + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + dfs_pos[s] = stack_sz + 1; + stack_sz++; + continue; + } + /* 's' is fully explored at this point */ + if (dfs_pos[s]) { + /* + * start -> ... -> s -> cur --. + * ^ | + * '----------' + * Case B: 's' is in the current DFS path. + */ + err = assign_header(env, dfs, cur, s); + if (err) + return err; + add_backedge(env, cur, s); + } else if (aux[s].loop_header == -1) { + /* + * start -> ... -> ... -> s -> ... -> end + * | ^ + * '---> cur ---' + * Case C: 's' is explored, not in the current DFS path, + * and not a part of any loop. + */ + } else if (dfs_pos[aux[s].loop_header]) { + /* + * .----------------------. + * v | + * start -> ... -> h -> ... -> ... -> s --' + * | ^ + * '---> cur ---' + * Case D: 's' is explored, not in current DFS path, + * but its innermost loop header is. + */ + err = assign_header(env, dfs, cur, aux[s].loop_header); + if (err) + return err; + } else { + /* + * case E: 's' is explored, not in current DFS path, + * its innermost loop header is not in current DFS path, + * hence 's' is another entry into the same loop. + */ + h = aux[s].loop_header; + mark_irreducible(env, h); + while (aux[h].loop_header != -1) { + h = aux[h].loop_header; + if (dfs_pos[h]) { + err = assign_header(env, dfs, cur, h); + if (err) + return err; + break; + } + mark_irreducible(env, h); + } + } + state[cur].next_succ++; + } while (stack_sz); + + return 0; +} + +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int idx; + + for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header) + if (aux[idx].loop_header == outer_header) + return true; + + return false; +} + +int bpf_compute_loops(struct bpf_verifier_env *env) +{ + int i, j, k, s, t, iloop, sloop, err = 0, len = env->prog->len; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct bpf_backedge *backedge; + struct loops_dfs dfs = {}; + struct bpf_iarray *succ; + struct bpf_loop *loop; + + dfs.dfs_pos = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + dfs.state = kvcalloc(len, sizeof(struct dfs_state), GFP_KERNEL_ACCOUNT); + dfs.stack = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + if (!dfs.dfs_pos || !dfs.state || !dfs.stack) { + err = -ENOMEM; + goto out; + } + for (i = 0; i < len; i++) + aux[i].loop_header = -1; + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_loops_in_subprog(env, &dfs, i); + if (err) + goto out; + } + /* find latches */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + /* + * In theory, the backedge->from and it's latch can reside in + * an inner loop of `i`, e.g.: + * + * 1: r7 += 1; + * 2: r6 += 1; + * if r7 == 2 goto 1b; + * if r6 < 2 goto 2b; + * + * For now, let's assume that latches are unknown for such cases. + * (GCC/LLVM handle this by inserting artificial cfg nodes). + */ + if (bpf_loop_at_index(env, backedge->from) != i) + continue; + find_dominating_conditions(env, backedge->from, backedge); + } + } + /* find exits */ + for (i = 0; i < len; i++) { + iloop = aux[i].loop ? i : aux[i].loop_header; + if (iloop < 0) + continue; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + /* + * Nothing left to record once the innermost loop of 'i' + * overflowed: the walk below would stop at it right away. + */ + if (aux[iloop].loop->exits_overflow) + break; + sloop = aux[s].loop ? s : aux[s].loop_header; + if (iloop == sloop) + continue; + if (bpf_is_nested_loop(env, sloop, iloop)) + continue; + /* + * At this point 'sloop' is either -1, an outer loop, + * or a loop in another branch of the loop hierarchy. + */ + for (t = iloop; t >= 0 && t != sloop; t = aux[t].loop_header) { + /* + * Account for the following configuration: + * + * tloop { + * iloop { + * ... i: goto s; + * } + * sloop { + * s: + * ... + * } + * } + */ + if (bpf_is_nested_loop(env, sloop, t)) + break; + /* + * Record edges i -> s as exits from tloop when: + * + * sloop { + * tloop { + * iloop { + * ... i: goto s; + * } + * } + * s: ... + * } + */ + /* Enclosing loops inherit the overflow, see below */ + if (aux[t].loop->exits_overflow) + break; + err = add_exit(aux[t].loop, i, s); + if (err) + goto out; + } + } + if (bpf_is_ldimm64(env->prog->insnsi + i)) + i++; + } + /* A nested loop with a truncated exit list can't be abstracted by SCEV */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop || !loop->exits_overflow) + continue; + for (t = aux[i].loop_header; t >= 0; t = aux[t].loop_header) + aux[t].loop->exits_overflow = true; + } + + if (env->log.level & BPF_LOG_LEVEL2) { + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + bpf_log(log, "loop at %d", i); + if (aux[i].loop_header >= 0) + bpf_log(log, ", nested in %d", aux[i].loop_header); + if (loop->irreducible) + bpf_log(log, ", irreducible"); + if (loop->exits_overflow) + bpf_log(log, ", too many exits"); + bpf_log(log, "\n"); + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + bpf_log(log, " backedge from %d", backedge->from); + for (k = 0; k < backedge->latches_cnt; k++) { + if (k == 0) + bpf_log(log, ", latch at "); + if (k == 1) + bpf_log(log, ", dominating exits at "); + if (k > 1) + bpf_log(log, ", "); + bpf_log(log, "%d", backedge->latches[k]); + } + bpf_log(log, "\n"); + } + for (j = 0; j < loop->exits_cnt; j++) + bpf_log(log, " exit from %d to %d\n", + loop->exits[j].from, loop->exits[j].to); + } + } + +out: + kvfree(dfs.dfs_pos); + kvfree(dfs.stack); + kvfree(dfs.state); + return err; +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5a29d05c523c..16b7758221b0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22872,13 +22872,17 @@ static void log_program(struct bpf_verifier_env *env) u64 pos, insn_pos; u32 i, j; - verbose(env, "Program dump (scc? idom insn#: live_regs_before):\n"); + verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n"); for (i = 0; i < insn_cnt; ++i) { verbose_linfo(env, i, " ; "); if (env->insn_aux_data[i].scc) verbose(env, "%3d ", env->insn_aux_data[i].scc); else verbose(env, " "); + if (env->insn_aux_data[i].loop_header >= 0) + verbose(env, "%3d ", env->insn_aux_data[i].loop_header); + else + verbose(env, " "); verbose(env, "%3d ", env->idoms[i]); verbose(env, "%3d: ", i); for (j = BPF_REG_0; j < BPF_REG_10; ++j) @@ -23108,6 +23112,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + ret = bpf_compute_loops(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; @@ -23260,6 +23268,8 @@ err_prep: release_btfs(env); err_free_env: bpf_free_subprog_jts(env); + if (env->insn_aux_data) + bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env); From f849c3a145eb243ef1c4b08ec320a2538c9e722b Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:43 -0700 Subject: [PATCH 413/660] bpf: Add bpf_set_reg_range() A utility function for setting a register's range and step information. Used by SCEV widening and clamping logic further in the series. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-25-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/verifier.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d1ebbfef5455..bc901d649472 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1820,6 +1820,8 @@ struct arg_access_info bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 range, u16 base, u16 step); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 16b7758221b0..585303d28690 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17273,6 +17273,18 @@ clear_id: return 0; } +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 range, u16 base, u16 step) +{ + reg->r64 = range; + reg->r32 = CNUM32_UNBOUNDED; + reg->step = step; + reg->base = base; + reg->var_off = tnum_unknown; + reg_bounds_sync(reg); /* this should infer the tnum alignment */ + return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); +} + /* check validity of 32-bit and 64-bit arithmetic operations */ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) { From bc2afa09c07e84973121053b4f7fd7a759faa71b Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:44 -0700 Subject: [PATCH 414/660] bpf: Add bpf_mark_reg_known_scalar() Provide a helper to set a register to a known scalar value for use outside verifier.c. To be used by SCEV widening logic. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-26-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/verifier.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bc901d649472..c7b7c8148205 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1353,6 +1353,7 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 585303d28690..743fc2f3c24b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2057,6 +2057,12 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) ___mark_reg_known(reg, imm); } +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm) +{ + __mark_reg_known(reg, imm); + reg->type = SCALAR_VALUE; +} + static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); From 6360b15f86ed9ba90a4ec0d27e4d97eab2e5ca54 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:45 -0700 Subject: [PATCH 415/660] bpf: Add bpf_reg_union() and bpf_reg_union_compatible() Utility functions to take a union of two registers' scalar values: merge the circular 32-bit and 64-bit bounds, tnums and base/step of two registers, then synchronize and validate the result. Leave id invalidation and type validation to the caller. Caller should use bpf_reg_union_compatible() to ensure that final union result is sensible. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-27-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 3 +++ kernel/bpf/verifier.c | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c7b7c8148205..1d0a0b64b1f3 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1823,6 +1823,9 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct cnum64 range, u16 base, u16 step); +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 743fc2f3c24b..f8e6b40c2121 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -17291,6 +17292,51 @@ int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); } +/* + * Checks if pointer and packet metadata matches between two registers. + * For use with bpf_reg_union(). + */ +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b) +{ + if (a->type != b->type) + return false; + if (a->type == SCALAR_VALUE) + return true; + return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) && + a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid; +} + +/* + * Extends acc's scalar range to include src's scalar range. + * Caller must ensure that other kinds of metadata is compatible. + * Caller must clear acc's scalar ID. + */ +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src) +{ + u16 base, step; + + if (!bpf_reg_union_compatible(acc, src)) { + verifier_bug(env, "union of incompatible registers"); + return -EFAULT; + } + acc->r64 = cnum64_union(acc->r64, src->r64); + acc->r32 = cnum32_union(acc->r32, src->r32); + acc->var_off = tnum_union(acc->var_off, src->var_off); + + /* Retain a common congruence if the bases agree modulo the gcd. */ + step = gcd(acc->step, src->step); + base = acc->base % step; + if (base != src->base % step) { + reg_step_reset(acc); + } else { + acc->base = base; + acc->step = step; + } + reg_bounds_sync(acc); + return reg_bounds_sanity_check(env, acc, "bpf_reg_union"); +} + /* check validity of 32-bit and 64-bit arithmetic operations */ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) { From 97d592e69d097c5ba5381165b21f08b28f831695 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:46 -0700 Subject: [PATCH 416/660] bpf: Add bpf_split_cur_state() SCEV widening needs an explicit checkpoint preserving the original loop-entry state, independently of ordinary state-cache heuristics. This commit extracts checkpoint creation logic from bpf_is_state_visited() into a separate utility function. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-28-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/states.c | 95 ++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1d0a0b64b1f3..711a4cb24c06 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1339,6 +1339,7 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_split_cur_state(struct bpf_verifier_env *env); int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_jmp_history(struct bpf_verifier_state *state); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 2f6a7620eb72..b1e2daefc264 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1262,11 +1262,61 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ } } +int bpf_split_cur_state(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state, *new; + struct bpf_verifier_state_list *new_sl; + struct list_head *head; + int insn_idx = cur->insn_idx; + int err; + + head = bpf_explored_state(env, insn_idx); + new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); + if (!new_sl) + return -ENOMEM; + env->total_states++; + env->explored_states_size++; + update_peak_states(env); + env->prev_jmps_processed = env->jmps_processed; + env->prev_insn_processed = env->insn_processed; + + /* forget precise markings we inherited, see __mark_chain_precision */ + if (env->bpf_capable) + mark_all_scalars_imprecise(env, cur); + + bpf_clear_singular_ids(env, cur); + + /* add new state to the head of linked list */ + new = &new_sl->state; + err = bpf_copy_verifier_state(new, cur); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + new->insn_idx = insn_idx; + verifier_bug_if(new->branches != 1, env, + "%s:branches_to_explore=%d insn %d", + __func__, new->branches, insn_idx); + err = maybe_enter_scc(env, new); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + + cur->parent = new; + cur->first_insn_idx = insn_idx; + cur->dfs_depth = new->dfs_depth + 1; + bpf_clear_jmp_history(cur); + list_add(&new_sl->node, head); + return 0; +} + int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) { - struct bpf_verifier_state_list *new_sl; struct bpf_verifier_state_list *sl; - struct bpf_verifier_state *cur = env->cur_state, *new; + struct bpf_verifier_state *cur = env->cur_state; bool force_new_state, add_new_state, loop; int n, err, states_cnt = 0; struct list_head *pos, *tmp, *head; @@ -1594,44 +1644,5 @@ miss: * When looping the sl->state.branches will be > 0 and this state * will not be considered for equivalence until branches == 0. */ - new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); - if (!new_sl) - return -ENOMEM; - env->total_states++; - env->explored_states_size++; - update_peak_states(env); - env->prev_jmps_processed = env->jmps_processed; - env->prev_insn_processed = env->insn_processed; - - /* forget precise markings we inherited, see __mark_chain_precision */ - if (env->bpf_capable) - mark_all_scalars_imprecise(env, cur); - - bpf_clear_singular_ids(env, cur); - - /* add new state to the head of linked list */ - new = &new_sl->state; - err = bpf_copy_verifier_state(new, cur); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - new->insn_idx = insn_idx; - verifier_bug_if(new->branches != 1, env, - "%s:branches_to_explore=%d insn %d", - __func__, new->branches, insn_idx); - err = maybe_enter_scc(env, new); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - - cur->parent = new; - cur->first_insn_idx = insn_idx; - cur->dfs_depth = new->dfs_depth + 1; - bpf_clear_jmp_history(cur); - list_add(&new_sl->node, head); - return 0; + return bpf_split_cur_state(env); } From 4c3a56544b65d50b68c48d5d7829feb455ac7ade Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:47 -0700 Subject: [PATCH 417/660] bpf: Add bpf_same_memory_origin() When dealing with loops like: for (ptr = ...; ptr < end_ptr; ptr += 8) ... Scalar evolution needs to know if ptr and end_ptr identify a same memory location (possibly with different offsets). If they do, it is sound to attempt to infer iteration bounds for such a loop. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-29-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/verifier.c | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 711a4cb24c06..6daa0c826f25 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1822,6 +1822,8 @@ struct arg_access_info bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b); int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct cnum64 range, u16 base, u16 step); bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f8e6b40c2121..bf0722752e76 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17280,6 +17280,55 @@ clear_id: return 0; } +/* + * Many checks done by this function are quite conservative. + * This is because main verification pass does not maintain + * enough information to track object identities for some + * of the interesting types, e.g. PTR_TO_MEM. + */ +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b) +{ + if (reg_a == reg_b) + return true; + /* Require matching flags and base types. */ + if (reg_a->type != reg_b->type) + return false; + /* NULL can't be compared to some base+offset pointer. */ + if (type_may_be_null(reg_a->type)) + return false; + + switch (base_type(reg_a->type)) { + case PTR_TO_STACK: + return reg_a->frameno == reg_b->frameno; + case PTR_TO_MAP_VALUE: + if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid) + return false; + if (reg_a->id && reg_a->id == reg_b->id) + return true; + /* + * A plain single-element array has one stable value address, + * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal(). + */ + if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY && + reg_a->map_ptr->max_entries == 1 && + !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP)) + return true; + /* + * The rules above can be simplified / relaxed if: + * - fresh IDs would always be assigned for map-value lookups; + * - direct map value loads would always have and ID of zero. + */ + return false; + case PTR_TO_MEM: + case PTR_TO_BUF: + return reg_a->id && reg_a->id == reg_b->id; + + default: + return false; + } +} + int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct cnum64 range, u16 base, u16 step) { From 1291ad5906933355fbe7b042ee5e3eedb5743e46 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:48 -0700 Subject: [PATCH 418/660] bpf: Add bpf_spill_size() A utility function counting the number of STACK_SPILL slots in a bpf_stat_state instance. For use by SCEV widening logic. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-30-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 9 +++++++++ kernel/bpf/verifier.c | 5 +---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6daa0c826f25..7fb1e315de48 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1420,6 +1420,15 @@ static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; } +static inline int bpf_spill_size(const struct bpf_stack_state *stack) +{ + int i, spill_size = 0; + + for (i = BPF_REG_SIZE - 1; i >= 0 && stack->slot_type[i] == STACK_SPILL; i--) + spill_size++; + return spill_size; +} + static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) { return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bf0722752e76..85971f9c09c4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4247,10 +4247,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); if (bpf_is_spilled_reg(ss)) { - u8 spill_size = 1; - - for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) - spill_size++; + u8 spill_size = bpf_spill_size(ss); if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { if (reg->type != SCALAR_VALUE) { From 2fa579694ddf3a96454669c764ca0c06dea50817 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:49 -0700 Subject: [PATCH 419/660] bpf: Record basic-block ends in insn_aux_data SCEV needs information about the program's basic-block structure. Piggyback on compute_predecessors() and set insn_aux_data->bb_end for an instruction if: - it has multiple successors, or - its sole successor has multiple predecessors, or - it is a jump instruction. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-31-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/loops.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7fb1e315de48..1458e50a1ad1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -741,6 +741,7 @@ struct bpf_insn_aux_data { bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ + bool bb_end; /* last instruction of a basic block */ u8 alu_state; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c index 8a6dc805e559..2568079c2945 100644 --- a/kernel/bpf/loops.c +++ b/kernel/bpf/loops.c @@ -5,8 +5,23 @@ #include #include +static bool is_cfg_jump(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + switch (class) { + case BPF_JMP: + case BPF_JMP32: + return opcode != BPF_CALL; + default: + return false; + } +} + static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_iarray *succ, *preds, **result; struct bpf_prog *prog = env->prog; u32 *num_preds, i, s, len = prog->len; @@ -54,6 +69,10 @@ static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) preds = result[s]; preds->items[preds->cnt++] = i; } + if (succ->cnt > 1 || + (succ->cnt == 1 && num_preds[succ->items[0]] > 1) || + is_cfg_jump(insn)) + aux[i].bb_end = true; if (bpf_is_ldimm64(insn)) i++; } From 60b0109ad195c8b131224cfe66cbd6277c8d47b4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:50 -0700 Subject: [PATCH 420/660] bpf: Allow precision backtracking between overlapping checkpoints SCEV integration logic might create two consecutive checkpoints at the same instruction w/o processing any instructions in between. This happens for loop-entry checkpoint -> regular checkpoint sequences, where the loop-entry checkpoint has to remain unchanged but the regular checkpoint is used for widening. This commit adapts bpf_mark_chain_precision() to support this. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-32-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 5 ++++- kernel/bpf/backtrack.c | 5 +++++ kernel/bpf/states.c | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1458e50a1ad1..963106328791 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -502,7 +502,10 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; - /* first and last insn idx of this verifier state */ + /* + * First and last insn idx of this verifier state. + * last_insn_idx is -1 if no instructions have been executed yet. + */ u32 first_insn_idx; u32 last_insn_idx; /* if this state is a backedge state then equal_state diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 0e38b9575328..43d5740034c0 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -888,6 +888,10 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } if (last_idx < 0) { + /* Consecutive checkpoints can have no instructions between them. */ + if (st->parent) + goto parent; + /* we are at the entry into subprog, which * is expected for global funcs, but only if * requested precise registers are R1-R5 @@ -952,6 +956,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return -EFAULT; } } +parent: st = st->parent; if (!st) break; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index b1e2daefc264..470bff8d8dc8 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1306,6 +1306,7 @@ int bpf_split_cur_state(struct bpf_verifier_env *env) } cur->parent = new; + cur->last_insn_idx = -1; cur->first_insn_idx = insn_idx; cur->dfs_depth = new->dfs_depth + 1; bpf_clear_jmp_history(cur); From 1501b536818b99a643842296a062019d221716f7 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:51 -0700 Subject: [PATCH 421/660] bpf: Compute scalar evolution expressions for loops Assign algebraic expressions to loop variables (registers and stack spills), describing how their values evolve across iterations. These summaries provide symbolic input for the subsequent widening patch. The algorithm draws on ideas from the following papers: - "Symbolic Evaluation of Chains of Recurrences for Loop Optimization" Robert A. van Engelen, 2000 - "The CR# Algebra and its Application in Loop Analysis and Optimization" Robert A. van Engelen, 2004 The analysis proceeds in two phases: - Compute expressions for one symbolic iteration: start each variable with a reference to its input value, apply instruction effects and join expressions where control-flow paths merge. - Convert the resulting backedge updates to recurrences at the header, then substitute these recurrences into expressions at the latch. Analyze loops innermost first. Within each loop, visit blocks in topological order (reverse postorder, with backedges ignored and nested loops collapsed). Treat each nested loop as an opaque operation: preserve its invariant values, forget the others and continue at its exits, without expanding its iterations. For example, using 64-bit arithmetic: 0: r7 = 5; 1: r6 = 0; 2: do { // header 3: if (r6 == 1) 4: r7 = 10; 5: r6 += 1; 6: } while (r6 < 3); // latch Expressions use symbolic inputs r6 and r7; the concrete values from lines 0-1 are supplied later by the main verifier. First phase: ------------ At each instruction, an environment maps register numbers to their current expressions. Transfer functions update this mapping, while joins merge the environments arriving along different paths. For illustration, follow each branch separately to the backedge: - At 3, take the path skipping 4: r6 = r6, r7 = r7. - At 5, transfer(r6 += 1): r6 = r6 -> r6 = (+ r6 1). - At 2, the first path contributes r6 = (+ r6 1), r7 = r7. - At 4, on the other path from 3, transfer(r7 = 10): r7 = r7 -> r7 = 10. - At 5, transfer(r6 += 1): r6 = r6 -> r6 = (+ r6 1). - At 2, join the two paths' contributions: join((+ r6 1), (+ r6 1)) = (+ r6 1) for r6; join(r7, 10) = (any r7 10) for r7. Equal expressions stay unchanged; different expressions become ANY alternatives. These summarize one iteration, without unrolling the loop. Second phase: ------------- The update r6 = (+ r6 1) says that each iteration adds 1 to r6. Repeating this update n times gives r6_entry + n, a linear recurrence. - At header 2, r6 = (+ r6 1) becomes (linear r6 1), meaning r6_entry + n, where n counts iterations from zero. - At latch 6, substitute this recurrence into (+ r6 1): (+ (linear r6 1) 1) simplifies to (linear (+ r6 1) 1), meaning r6_entry + 1 + n. - At both points, r7 retains (any r7 10): either its entry value or 10. ANY records alternatives, not numeric ranges or the path conditions selecting them. Unchanged variables retain their inputs. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-33-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 7 + kernel/bpf/Makefile | 2 +- kernel/bpf/scev.c | 1640 ++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 9 + 4 files changed, 1657 insertions(+), 1 deletion(-) create mode 100644 kernel/bpf/scev.c diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 963106328791..52f72e92eb0c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -745,6 +745,7 @@ struct bpf_insn_aux_data { bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ bool bb_end; /* last instruction of a basic block */ + bool need_scev; u8 alu_state; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. @@ -1000,6 +1001,7 @@ struct bpf_scc_info { }; struct bpf_liveness; +struct scev; struct bpf_fd_array { union { @@ -1164,6 +1166,7 @@ struct bpf_verifier_env { struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; int *idoms; + struct scev *scev; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1964,4 +1967,8 @@ int bpf_compute_loops(struct bpf_verifier_env *env); int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); +int bpf_init_scev(struct bpf_verifier_env *env); +void bpf_free_scev(struct bpf_verifier_env *env); +int bpf_compute_scev(struct bpf_verifier_env *env); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 6210601eb40b..d61cbb59ad08 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o scev.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c new file mode 100644 index 000000000000..2d2a84156180 --- /dev/null +++ b/kernel/bpf/scev.c @@ -0,0 +1,1640 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include + +#define REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) +#define UNKNOWN_EXPR_ID 0 +#define OPAQUE_EXPR_ID 1 + +/* + * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding. + * For scalar evolution purpose we want to reuse most of the opcode definitions, + * but also add a few custom operations (REG and IMM). + * Use the enum below to uniformly represent all operations relevant for SCEV. + */ +enum expr_op { + /* leave range 0..255 for standard bpf opcodes */ + UNKNOWN = 256, /* start custom opcodes from the second byte */ + REG, + IMM, + SDIV, SMOD, + SEXT8, SEXT16, SEXT32, + ZEXT8, ZEXT16, ZEXT32, + BSWAP16, BSWAP32, BSWAP64, + /* during the loop body execution the value can be either of param[0] or param[1] */ + ANY, + /* some value that verifier is not going to track precisely */ + OPAQUE, + /* '*(u8/16/32 *)(r10 + X) = Y' writes define slot contents only partially */ + SPILL8, SPILL16, SPILL32, + /* + * SCEV expression corresponding to linear equation 'param[0] + param[1] * k', + * where k is a loop iteration number. Loop here refers to innermost loop + * containing instruction associated with this expression, as returned by + * bpf_loop_at_index(). + */ + LINEAR_SCEV, +}; + +struct expr { + u32 op; + union { + u32 params[2]; + s64 imm; + }; +}; + +struct expr_bucket { + u32 cnt; + u32 cap; + u32 ids[]; +}; + +struct env { + bool empty; + u32 reg2expr[REGS_NUM]; + u32 reg2scev[REGS_NUM]; +}; + +struct insn_envs { + u32 cnt; + struct { + int loop_header; + struct env *env; + } entries[]; +}; + +#define EXPR_STACK_DEPTH 8 + +struct expr_stack_elt { + u32 id:28; + u32 pre:1; + u32 next_param:2; +}; + +DEFINE_MIN_HEAP(int, scev_worklist); + +struct scev { + /* + * Expressions are identified by id, exprs_ht ensures that + * each expression exists as a unique instance. + * This allows for fast equivalence check: id1 === id2. + */ + struct expr_bucket **exprs_ht; + struct scev_worklist worklist; + struct insn_envs **envs; + struct expr *exprs; + /* + * Loops are analyzed one by one, this array keeps track if a particular + * instruction was visited on a current pass. + */ + u32 *discovered; + u32 exprs_ht_cnt; + int exprs_cnt; + int exprs_cap; + int envs_cnt; + int stack_sz; + struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH]; + u32 ids_buf[EXPR_STACK_DEPTH]; +}; + +static u32 expr_hash(struct expr *e) +{ + return jhash_3words(e->op, e->params[0], e->params[1], 0); +} + +static int expr_eq(struct expr *a, struct expr *b) +{ + return a->op == b->op && a->imm == b->imm; +} + +static int add_expr(struct scev *scev, struct expr e) +{ + struct expr_bucket *bucket; + u32 i, id, hash, new_cap; + void *tmp; + + hash = expr_hash(&e) & (scev->exprs_ht_cnt - 1); + bucket = scev->exprs_ht[hash]; + + if (bucket) { + for (i = 0; i < bucket->cnt; i++) { + id = bucket->ids[i]; + if (expr_eq(&e, &scev->exprs[id])) + return id; + } + } + + if (!bucket || bucket->cap == bucket->cnt) { + new_cap = bucket ? bucket->cap * 2 : 4; + bucket = kvrealloc(bucket, sizeof(*bucket) + sizeof(u32) * new_cap, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!bucket) + return -ENOMEM; + scev->exprs_ht[hash] = bucket; + bucket->cap = new_cap; + } + + if (scev->exprs_cnt == scev->exprs_cap) { + new_cap = scev->exprs_cap ? scev->exprs_cap * 2 : 256; + tmp = kvrealloc(scev->exprs, sizeof(struct expr) * new_cap, GFP_KERNEL_ACCOUNT); + if (!tmp) + return -ENOMEM; + scev->exprs = tmp; + scev->exprs_cap = new_cap; + } + + id = scev->exprs_cnt++; + scev->exprs[id] = e; + bucket->ids[bucket->cnt++] = id; + return id; +} + +static int expr2(struct scev *scev, u32 op, int a, int b) +{ + if (a < 0) + return a; + if (b < 0) + return b; + return add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); +} + +static int expr1(struct scev *scev, u32 op, int a) +{ + if (a < 0) + return a; + return expr2(scev, op, a, 0); +} + +static int expr0(struct scev *scev, u32 op) +{ + return expr2(scev, op, 0, 0); +} + +static int imm_expr(struct scev *scev, s64 value) +{ + return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); +} + +static bool same_exprs(struct scev *scev, int id_a, int id_b) +{ + return id_a == id_b; +} + +static bool is_imm(struct scev *scev, int id, s64 *imm) +{ + if (scev->exprs[id].op != IMM) + return false; + *imm = scev->exprs[id].imm; + return true; +} + +static bool is_op(struct scev *scev, int id, u32 op) +{ + if (scev->exprs[id].op != op) + return false; + return true; +} + +static bool is_unop(struct scev *scev, enum expr_op op, int id, u32 *p0) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + return true; +} + +static bool is_binop(struct scev *scev, enum expr_op op, int id, u32 *p0, u32 *p1) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + *p1 = scev->exprs[id].params[1]; + return true; +} + +static bool is_reg(struct scev *scev, int id, u32 *reg) +{ + return is_unop(scev, REG, id, reg); +} + +static bool is_add(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, BPF_ADD, id, left, right); +} + +static bool is_zext32(struct scev *scev, int id, u32 *left) +{ + return is_unop(scev, ZEXT32, id, left); +} + +static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, ANY, id, left, right); +} + +static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope) +{ + return is_binop(scev, LINEAR_SCEV, id, base, slope); +} + +static bool is_opaque(struct scev *scev, int id) +{ + return is_op(scev, id, OPAQUE); +} + +static void log_reg(struct bpf_verifier_env *env, u32 reg) +{ + if (reg < MAX_BPF_REG) + bpf_log(&env->log, "r%d", reg); + else + bpf_log(&env->log, "*fp%d", (MAX_BPF_REG - reg - 1) * 8); +} + +static const char *op_str(u32 op) +{ + switch (op) { + case BPF_ADD: return "+"; + case BPF_SUB: return "-"; + case BPF_MUL: return "*"; + case BPF_DIV: return "/"; + case SDIV: return "s/"; + case BPF_OR: return "|"; + case BPF_AND: return "&"; + case BPF_LSH: return "<<"; + case BPF_RSH: return ">>"; + case BPF_NEG: return "-"; + case BPF_MOD: return "%"; + case SMOD: return "s%"; + case BPF_XOR: return "^"; + case BPF_ARSH: return "s>>"; + case SEXT8: return "sext8"; + case SEXT16: return "sext16"; + case SEXT32: return "sext32"; + case ZEXT8: return "zext8"; + case ZEXT16: return "zext16"; + case ZEXT32: return "zext32"; + case BSWAP16: return "bswap16"; + case BSWAP32: return "bswap32"; + case BSWAP64: return "bswap64"; + case SPILL8: return "spill8"; + case SPILL16: return "spill16"; + case SPILL32: return "spill32"; + case ANY: return "any"; + case LINEAR_SCEV: return "linear"; + } + return NULL; +} + +static u32 op_params_num(u32 op) +{ + switch (op) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case SDIV: + case SMOD: + case ANY: + case LINEAR_SCEV: + return 2; + case BPF_NEG: + case SEXT8: + case SEXT16: + case SEXT32: + case ZEXT8: + case ZEXT16: + case ZEXT32: + case BSWAP16: + case BSWAP32: + case BSWAP64: + case SPILL8: + case SPILL16: + case SPILL32: + return 1; + case REG: + case IMM: + case OPAQUE: + return 0; + default: + return 0; + } +} + +static bool expr_stack_push(struct scev *scev, u32 id) +{ + if (scev->stack_sz >= EXPR_STACK_DEPTH) + return false; + scev->expr_stack[scev->stack_sz].id = id; + scev->expr_stack[scev->stack_sz].pre = true; + scev->expr_stack[scev->stack_sz].next_param = 0; + scev->stack_sz++; + return true; +} + +enum { + PRE = BIT(1), POST = BIT(2), DEPTH_LIMIT = BIT(3) +}; + +static bool expr_next(struct scev *scev, u32 *id, u32 *order) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 num_params; + + if (scev->stack_sz == 0) + return false; + + elt = &scev->expr_stack[scev->stack_sz - 1]; + *id = elt->id; + *order = 0; + expr = &scev->exprs[elt->id]; + num_params = op_params_num(expr->op); + if (elt->pre) { + elt->pre = false; + *order = PRE; + return true; + } + if (elt->next_param == num_params) { + *order = POST; + scev->stack_sz--; + return true; + } + if (scev->stack_sz == EXPR_STACK_DEPTH) { + *order = POST | DEPTH_LIMIT; + scev->stack_sz--; + return true; + } + expr_stack_push(scev, expr->params[elt->next_param]); + elt->next_param++; + return expr_next(scev, id, order); +} + +static void log_expr(struct bpf_verifier_env *env, u32 id) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct expr *expr; + const char *str; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->stack_sz > 1) + bpf_log(log, " "); + expr = &scev->exprs[id]; + switch (expr->op) { + case UNKNOWN: + if (order & PRE) + bpf_log(log, "?"); + break; + case OPAQUE: + if (order & PRE) + bpf_log(log, "_"); + break; + case REG: + if (order & PRE) + log_reg(env, expr->params[0]); + break; + case IMM: + if (order & PRE) + bpf_log(log, "%lld", expr->imm); + break; + default: + if (order & PRE) { + str = op_str(expr->op); + bpf_log(log, "("); + if (str) + bpf_log(log, "%s", str); + else + bpf_log(log, "bad-expr-op %x", expr->op); + } + if (order & DEPTH_LIMIT) + bpf_log(log, "..."); + if (order & POST) + bpf_log(log, ")"); + } + } +} + +static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + u16 live_regs = aux[insn_idx].live_regs_before; + + return reg < MAX_BPF_REG + ? (BIT(reg) & live_regs) + : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before); +} + +static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + bool printed_some = false; + bool is_self_scev; + bool is_self_reg; + int i, r; + + if (e->empty) { + bpf_log(log, " \n"); + return; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!reg_alive_at(env, i, insn_idx)) + continue; + is_self_reg = is_reg(scev, e->reg2expr[i], &r) && i == r; + is_self_scev = is_reg(scev, e->reg2scev[i], &r) && i == r; + if (is_self_reg && is_self_scev) + continue; + printed_some = true; + bpf_log(log, " "); + log_reg(env, i); + bpf_log(log, "="); + log_expr(env, e->reg2expr[i]); + if (e->reg2expr[i] != UNKNOWN_EXPR_ID) { + bpf_log(log, " / "); + log_expr(env, e->reg2scev[i]); + } + bpf_log(log, "\n"); + } + if (!printed_some) + bpf_log(log, " \n"); +} +static struct env *find_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 i; + + if (!envs) + return NULL; + for (i = 0; i < envs->cnt; i++) + if (envs->entries[i].loop_header == loop_header) + return envs->entries[i].env; + return NULL; +} + +static struct env *find_header_env(struct scev *scev, int header) +{ + return find_loop_env(scev, header, header); +} + +static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 cnt = envs ? envs->cnt : 0; + struct insn_envs *tmp; + struct env *e; + + e = find_loop_env(scev, loop_header, insn_idx); + if (e) + return e; + + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); + if (!e) + return NULL; + + tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT); + if (!tmp) { + kfree(e); + return NULL; + } + + e->empty = true; + tmp->entries[cnt].loop_header = loop_header; + tmp->entries[cnt].env = e; + tmp->cnt = cnt + 1; + scev->envs[insn_idx] = tmp; + return e; +} + +static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) +{ + if (id < 0) + return id; + e->reg2expr[reg] = id; + return 0; +} + +static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, int insn_idx) +{ + struct scev *scev = env->scev; + int i, err; + + /* + * Initial assignments are of form rX -> rX are needed as seeds for subsequent + * work queue pop / transfer() calls. Because e->empty is true, the first backedge + * join() will overwrite the them. compute_header_scevs() accounts for register + * states at loop entry. + */ + for (i = 0; i < REGS_NUM; i++) { + err = replace_reg(scev, e, i, expr1(scev, REG, i)); + if (err) + return err; + } + return 0; +} + +static void forget_call_regs(struct env *e) +{ + int i; + + for (i = BPF_REG_0; i <= BPF_REG_5; i++) + e->reg2expr[i] = UNKNOWN_EXPR_ID; +} + +static u32 spill_spi(struct bpf_insn *insn) +{ + return -insn->off / BPF_REG_SIZE - 1; +} + +static u32 off_to_reg(struct bpf_insn *insn) +{ + return spill_spi(insn) + MAX_BPF_REG; +} + +static bool is_spill_off(int off) +{ + return off % BPF_REG_SIZE == 0 && + off <= -BPF_REG_SIZE && + off >= -MAX_BPF_STACK_JIT; +} + +static void mark_opaque(struct env *e, int off, int size) +{ + int b, spi; + + for (b = off; b < off + size; b++) { + if (b >= 0 || b < -MAX_BPF_STACK_JIT) + continue; + spi = (-b - 1) / BPF_REG_SIZE; + e->reg2expr[MAX_BPF_REG + spi] = OPAQUE_EXPR_ID; + } +} + +static int mk_spill(struct scev *scev, u8 size, int id) +{ + switch (size) { + case BPF_B: return expr1(scev, SPILL8, id); + case BPF_H: return expr1(scev, SPILL16, id); + case BPF_W: return expr1(scev, SPILL32, id); + case BPF_DW: return id; + } + return UNKNOWN_EXPR_ID; +} + +static int mk_fill(struct scev *scev, int id, u8 code) +{ + bool sx = BPF_MODE(code) == BPF_MEMSX; + +#ifdef __BIG_ENDIAN + /* A narrow fill from a DW spill reads high bits on big-endian. */ + if (BPF_SIZE(code) != BPF_DW) + return OPAQUE_EXPR_ID; +#endif + + switch (BPF_SIZE(code)) { + case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id); + case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id); + case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id); + case BPF_DW: return sx ? UNKNOWN_EXPR_ID : id; /* DW sign-extended load is invalid */ + } + return UNKNOWN_EXPR_ID; +} + +static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int id) +{ + u8 size = BPF_SIZE(insn->code); + + if (insn->dst_reg != BPF_REG_FP) + return 0; + if (is_spill_off(insn->off)) + return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id)); + mark_opaque(e, insn->off, bpf_size_to_bytes(size)); + return 0; +} + +static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn) +{ + int id; + + if (insn->src_reg == BPF_REG_FP && is_spill_off(insn->off)) + id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code); + else + id = OPAQUE_EXPR_ID; + return replace_reg(scev, e, insn->dst_reg, id); +} + +static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) +{ + const bool little_endian = htons(0x3412) == 0x1234; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u32 *reg2expr = e->reg2expr; + u8 class = BPF_CLASS(insn->code); + u8 x_or_k = BPF_SRC(insn->code); + u8 opcode = BPF_OP(insn->code); + u8 mode = BPF_MODE(insn->code); + u32 dst = insn->dst_reg; + u32 src = insn->src_reg; + u32 op, sext; + bool need_bswap; + int i, id; + s64 imm; + + switch (class) { + case BPF_ALU: + case BPF_ALU64: + switch (opcode) { + case BPF_MOV: + switch (insn->off) { + case 0: sext = 0; break; + case 8: sext = SEXT8; break; + case 16: sext = SEXT16; break; + case 32: sext = SEXT32; break; + default: + goto mark_dst_unknown; + } + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0 && insn->off == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + if (sext) + id = expr1(scev, sext, id); + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + if (opcode == BPF_DIV && insn->off == 1) + op = SDIV; + else if (opcode == BPF_MOD && insn->off == 1) + op = SMOD; + else if (insn->off == 0) + op = opcode; + else + goto mark_dst_unknown; + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + id = expr2(scev, op, reg2expr[dst], id); + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_NEG: + if (src == 0 && insn->off == 0 && insn->imm == 0) + id = expr1(scev, BPF_NEG, reg2expr[dst]); + else + goto mark_dst_unknown; + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_END: + if (src != 0 || insn->off != 0 || (class == BPF_ALU64 && x_or_k != 0)) + goto mark_dst_unknown; + + need_bswap = class == BPF_ALU64 || ((x_or_k == BPF_TO_LE) != little_endian); + switch (insn->imm) { + case 16: op = need_bswap ? BSWAP16 : ZEXT16; break; + case 32: op = need_bswap ? BSWAP32 : ZEXT32; break; + case 64: op = need_bswap ? BSWAP64 : 0; break; + default: + goto mark_dst_unknown; + } + + id = reg2expr[dst]; + if (op) + id = expr1(scev, op, reg2expr[dst]); + + return replace_reg(scev, e, dst, id); + default: + goto mark_dst_unknown; + } + break; + case BPF_LDX: + switch (mode) { + case BPF_MEM: + case BPF_MEMSX: + return maybe_load_fp(scev, e, insn); + default: + goto mark_dst_unknown; + } + case BPF_STX: + switch (mode) { + case BPF_MEM: + return maybe_store_fp(scev, e, insn, reg2expr[src]); + case BPF_ATOMIC: + if (insn->imm == BPF_LOAD_ACQ) + return maybe_load_fp(scev, e, insn); + if (insn->imm == BPF_STORE_REL) { + return maybe_store_fp(scev, e, insn, reg2expr[src]); + } + /* + * verifier does not track other atomic ops precisely, + * hence mark the results as opaque. + */ + if (insn->dst_reg == BPF_REG_FP) + mark_opaque(e, insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (insn->imm == BPF_CMPXCHG) + return replace_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID); + if (insn->imm & BPF_FETCH) + return replace_reg(scev, e, src, OPAQUE_EXPR_ID); + break; + } + break; + case BPF_ST: + if (insn->dst_reg == BPF_REG_FP) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + return maybe_store_fp(scev, e, insn, id); + } + break; + case BPF_JMP: + case BPF_JMP32: + if (opcode == BPF_CALL) + forget_call_regs(e); + /* for non-CALL there are no changes in register states */ + break; + case BPF_LD: + switch (mode) { + case BPF_IMM: + /* rX = imm ll */ + if (BPF_SIZE(insn->code) == BPF_DW && insn->src_reg == 0) { + imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; + return replace_reg(scev, e, dst, imm_expr(scev, imm)); + } + /* map, map value, BTF id, function */ + if (BPF_SIZE(insn->code) == BPF_DW) + return replace_reg(scev, e, dst, OPAQUE_EXPR_ID); + goto mark_dst_unknown; + case BPF_ABS: + case BPF_IND: + forget_call_regs(e); + break; + default: + goto mark_dst_unknown; + } + break; + default: + /* unknown instruction, nuke state */ + for (i = 0; i < REGS_NUM; i++) + reg2expr[i] = UNKNOWN_EXPR_ID; + break; + } + return 0; + +mark_dst_unknown: + reg2expr[dst] = UNKNOWN_EXPR_ID; + return 0; +} + +/* + * Construct minimal 'ANY' expression by traversing 'a' and 'b' + * and accumulating non-duplicated non-ANY entries. + */ +static int mk_any(struct scev *scev, int a, int b) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 i, j, ids_buf_sz; + u32 roots[2] = {a, b}; + int id; + + if (a < 0) + return a; + if (b < 0) + return b; + ids_buf_sz = 0; + for (i = 0; i < ARRAY_SIZE(roots); i++) { + scev->stack_sz = 0; + expr_stack_push(scev, roots[i]); + while (scev->stack_sz) { + elt = &scev->expr_stack[--scev->stack_sz]; + id = elt->id; + expr = &scev->exprs[elt->id]; + if (expr->op == ANY) { + if (!expr_stack_push(scev, expr->params[1]) || + !expr_stack_push(scev, expr->params[0])) + return UNKNOWN_EXPR_ID; + } else { + for (j = 0; j < ids_buf_sz; j++) { + if (same_exprs(scev, scev->ids_buf[j], id)) + goto next; + } + if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf)) + return UNKNOWN_EXPR_ID; + scev->ids_buf[ids_buf_sz++] = id; + } +next:; + } + } + if (WARN_ON(ids_buf_sz == 0)) + return -EFAULT; + id = scev->ids_buf[0]; + for (i = 1; i < ids_buf_sz; i++) { + id = expr2(scev, ANY, id, scev->ids_buf[i]); + if (id < 0) + return id; + } + return id; +} + +static int join(struct scev *scev, struct env *acc, struct env *cur) +{ + int i, id; + + if (acc->empty) { + memcpy(acc, cur, sizeof(*acc)); + acc->empty = false; + return 0; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { + id = mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]); + if (id < 0) + return id; + acc->reg2expr[i] = id; + } + } + return 0; +} + +/* Mark any register modified in 'e' as unknown in 'acc'. */ +static void forget_modified_regs(struct scev *scev, struct env *acc, struct env *e) +{ + u32 i, reg; + + for (i = 0; i < REGS_NUM; i++) { + if (e && is_reg(scev, e->reg2expr[i], ®) && reg == i) + continue; + acc->reg2expr[i] = UNKNOWN_EXPR_ID; + } +} + +/* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */ +static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[header].loop; + struct scev *scev = env->scev; + int i, idx, h; + + acc->empty = false; + forget_modified_regs(scev, acc, find_header_env(scev, header)); + for (i = 0; i < loop->exits_cnt; i++) { + idx = loop->exits[i].from; + h = bpf_loop_at_index(env, idx); + /* + * In order to account for situations like this: + * + * header: + * do { + * saved = rX; + * rX = -1000; + * do { // H + * if (random()) + * goto out; // E + * } while (inner_again); + * rX = saved; + * } while (outer_again); + * out: + * + * For exits originating in nested loops check that rX is invariant + * both at the header (H) and at exit (E). + */ + while (h != header) { + forget_modified_regs(scev, acc, find_header_env(scev, h)); + forget_modified_regs(scev, acc, find_loop_env(scev, h, idx)); + idx = h; + h = aux[h].loop_header; + } + /* + * For exits originating at the same loop the header is already accounted for, + * check only the exit environment. + */ + forget_modified_regs(scev, acc, find_loop_env(scev, header, idx)); + } +} + +/* Order worklist in reverse post-order. */ +static bool reverse_postorder_less(const void *a, const void *b, void *arg) +{ + const int *rank = arg; + + return rank[*(const int *)a] > rank[*(const int *)b]; +} + +static const struct min_heap_callbacks worklist_cb = { + .less = reverse_postorder_less, +}; + +static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx) +{ + struct scev *scev = env->scev; + struct scev_worklist *worklist = &scev->worklist; + u32 tag = (u32)loop_header + 1; + + if (scev->discovered[idx] == tag) + return 0; + + if (min_heap_full_inline(worklist)) { + size_t size = worklist->size ? worklist->size * 2 : 16; + int *data; + + data = kvrealloc(worklist->data, array_size(size, sizeof(*data)), + GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + worklist->data = data; + worklist->size = size; + } + if (!min_heap_push_inline(worklist, &idx, &worklist_cb, env->cfg.postorder_nums)) + return -EFAULT; + + scev->discovered[idx] = tag; + return 0; +} + +static bool reg_alive_at_succ(struct bpf_verifier_env *env, u32 r, u32 insn_idx) +{ + struct bpf_iarray *succ = bpf_insn_successors(env, insn_idx); + u32 i; + + for (i = 0; i < succ->cnt; i++) + if (reg_alive_at(env, r, succ->items[i])) + return true; + return false; +} + +enum changes_at { + LOG_AT_TRANSFER, + LOG_AT_JOIN, +}; + +static void log_env_changes(struct bpf_verifier_env *env, enum changes_at at, + struct env *old, struct env *new, int insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + u64 null_pos = log->end_pos; + u32 old_id, new_id, regno; + bool any_changes = false; + bool is_self_scev; + u64 len; + int r; + + bpf_log(log, "%s %4d: ", at == LOG_AT_TRANSFER ? "t" : "j", insn_idx); + bpf_verbose_insn(env, &env->prog->insnsi[insn_idx]); + len = log->end_pos - null_pos; + bpf_log(log, "%*s", max(37 - (int)len, 1), " "); + bpf_log(log, " ; "); + for (r = 0; r < REGS_NUM; r++) { + old_id = old->reg2expr[r]; + new_id = new->reg2expr[r]; + if (at == LOG_AT_TRANSFER && !reg_alive_at_succ(env, r, insn_idx)) + continue; + if (at == LOG_AT_JOIN && !reg_alive_at(env, r, insn_idx)) + continue; + if (same_exprs(scev, old_id, new_id)) + continue; + is_self_scev = is_reg(scev, new_id, ®no) && r == regno; + if (old->empty && is_self_scev) + continue; + if (any_changes) + bpf_log(log, ", "); + log_reg(env, r); + bpf_log(log, " "); + log_expr(env, old_id); + bpf_log(log, " -> "); + log_expr(env, new_id); + any_changes = true; + } + if (old->empty && !any_changes) + bpf_log(log, ""); + bpf_log(log, "\n"); + if (!old->empty && !any_changes) + bpf_vlog_reset(log, null_pos); +} + +static bool is_probe_read_helper(u32 func_id) +{ + return func_id == BPF_FUNC_probe_read || + func_id == BPF_FUNC_probe_read_kernel || + func_id == BPF_FUNC_probe_read_user || + func_id == BPF_FUNC_probe_read_str || + func_id == BPF_FUNC_probe_read_kernel_str || + func_id == BPF_FUNC_probe_read_user_str; +} + +/* + * If instruction is an indirect write to stack, invalidate SCEVs for spi's + * that this instruction can touch. + */ +static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u8 class = BPF_CLASS(insn->code); + const unsigned long *mask; + u32 spi, reg; + bool opaque; + + /* Direct fp stores are fine. */ + if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP) + return; + + mask = bpf_may_write_mask(env, idx); + opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm); + for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) { + reg = MAX_BPF_REG + spi; + if (cur_env->reg2expr[reg] == UNKNOWN_EXPR_ID) + continue; + replace_reg(scev, cur_env, reg, opaque ? OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID); + } +} + +/* Find the topmost loop header containing idx inside cur_header, or -1 if none. */ +static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int header; + + for (header = bpf_loop_at_index(env, idx); header >= 0; + header = aux[header].loop_header) + if (aux[header].loop_header == cur_header) + return header; + + return -1; +} + +/* + * Join cur_env into the successor's environment and schedule their traversal. + * Map successors in nested loops to their topmost nested header. + * Ignore successors outside the current loop and its nested loops. + */ +static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ_idx, + struct env *old_env, struct env *cur_env) +{ + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + struct scev *scev = env->scev; + int err, nested_header; + struct env *succ_env; + + /* + * There are several possibilities for a successor: + * - succ_idx can be a part of a loop outside of the cur_header's loop, + * such edges are ignored. + * - succ_idx can be a part of the same loop as cur_header, + * for such edges succ_idx environment is updated: + * e[succ_idx] = join(e[succ_idx], cur_env) + * - succ_idx can be a part of some loop inner to cur_header, + * in such a case there exists some loop header H, + * such that H.loop_header == cur_header + * and H is the same as succ_idx's loop or contains it. + */ + if (bpf_loop_at_index(env, succ_idx) != cur_header) { + /* + * topmost_nested_loop() either finds H or returns -1, + * in case if succ_idx is a part of a loop outer to cur_header. + */ + nested_header = topmost_nested_loop(env, succ_idx, cur_header); + if (nested_header < 0) + return 0; + succ_idx = nested_header; + } + succ_env = get_loop_env(scev, cur_header, succ_idx); + if (!succ_env) + return -ENOMEM; + if (log_level2) + memcpy(old_env, succ_env, sizeof(*old_env)); + err = join(scev, succ_env, cur_env); + if (err) + return err; + err = worklist_push(env, cur_header, succ_idx); + if (err) + return err; + if (log_level2) + log_env_changes(env, LOG_AT_JOIN, old_env, succ_env, succ_idx); + return 0; +} + +/* Unsupported loops get only an unknown header environment. */ +static bool can_compute_loop_scev(const struct bpf_loop *loop) +{ + return !loop->irreducible && !loop->exits_overflow; +} + +static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) +{ + struct scev_worklist *worklist = &env->scev->worklist; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *cur_loop = aux[cur_header].loop; + struct env *header_env; + struct scev *scev = env->scev; + struct bpf_loop *nested_loop; + struct env *cur_env = NULL; + struct env *old_env = NULL; + struct bpf_iarray *succ; + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + int s, i, err, idx, succ_idx; + + if (log_level2) + bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); + + /* + * For irreducible loops, and for loops nesting a loop with a truncated + * exit list, just assume that everything is clobbered for now. + */ + if (!can_compute_loop_scev(cur_loop)) { + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + return -ENOMEM; + /* The freshly allocated environment has all expressions unknown. */ + header_env->empty = false; + return 0; + } + + cur_env = kzalloc(sizeof(*cur_env), GFP_KERNEL_ACCOUNT); + if (!cur_env) + goto nomem; + if (log_level2) { + old_env = kzalloc(sizeof(*old_env), GFP_KERNEL_ACCOUNT); + if (!old_env) + goto nomem; + } + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + goto nomem; + err = setup_initial_loop_env(env, header_env, cur_header); + if (err) + goto out; + err = worklist_push(env, cur_header, cur_header); + if (err) + goto out; + + while (worklist->nr) { + idx = *min_heap_peek_inline(worklist); + min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums); + + /* join_successor() maps nested-loop entries to their representative header. */ + nested_loop = idx != cur_header ? aux[idx].loop : NULL; + memcpy(cur_env, find_loop_env(scev, cur_header, idx), sizeof(*cur_env)); + if (nested_loop) { + /* + * Process nested loop as a single instruction by + * forgetting anything non-invariant in the nested loop + */ + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + forget_non_invariants(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + /* + * Treat nested loop exits as successors, + * join cur_env into successor's envs. + */ + for (i = 0; i < nested_loop->exits_cnt; i++) { + s = nested_loop->exits[i].to; + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } else { + /* + * Iterate instructions within a single basic block + * starting at 'idx' mutating 'cur_env'. + */ + for (;;) { + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + err = transfer(env, cur_env, idx); + if (err) + goto out; + reset_scevs_at_indirect_writes(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + succ = bpf_insn_successors(env, idx); + if (succ->cnt != 1) + break; + succ_idx = succ->items[0]; + if (aux[idx].bb_end || aux[succ_idx].need_scev || + bpf_loop_at_index(env, succ_idx) != cur_header) + break; + idx = succ_idx; + } + /* Join cur_env into basic block successor's envs. */ + iarray_for_each(s, succ) { + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } + } + + err = 0; +out: + kfree(cur_env); + kfree(old_env); + return err; +nomem: + err = -ENOMEM; + goto out; +} + +static void mark_latches(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop; + int len = env->prog->len; + int i, j, k, from; + + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + aux[i].need_scev = true; + if (!can_compute_loop_scev(loop)) + continue; + /* + * Make sure to only set .need_scev for exits and latches + * belonging to the loop at `i`. If can_compute_loop_scev() + * fails for a loop, it should hide it's instructions from + * compute_insn_scevs(). + */ + for (j = 0; j < loop->backedges_cnt; j++) + for (k = 0; k < loop->backedges[j].latches_cnt; k++) + aux[loop->backedges[j].latches[k]].need_scev = true; + for (j = 0; j < loop->exits_cnt; j++) { + from = loop->exits[j].from; + if (bpf_loop_at_index(env, from) == i) + aux[from].need_scev = true; + } + } +} + +static bool is_any_imm_reg_opaque(struct scev *scev, u32 id) +{ + u32 l, r, reg, order; + s64 imm; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || + is_imm(scev, id, &imm) || + is_reg(scev, id, ®) || + is_opaque(scev, id) || + is_any(scev, id, &l, &r)) + continue; + return false; + } + return true; +} + +/* + * Can implement explicit stack version, but it is harder to read. + * Stick with recursive version for now. + */ +static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + struct expr expr; + int p0, p1, id; + + if (lvl >= EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + + expr = scev->exprs[root]; /* snapshot the expr before potential realloc */ + switch (op_params_num(expr.op)) { + case 0: + id = root; + break; + case 1: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + id = expr1(scev, expr.op, p0); + break; + case 2: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + p1 = transform_expr_once(scev, lvl + 1, expr.params[1], priv, fn); + id = expr2(scev, expr.op, p0, p1); + break; + } + return id < 0 ? id : fn(scev, id, priv); +} + +static int transform_expr(struct scev *scev, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + int i = 0, id_old, id_new = root; + + do { + /* + * The depth limit is somewhat arbitrary and is essentially a failsafe. + * oscillation should not really happen. Still, it seems harsh to error out. + */ + if (i++ == 4 * EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + id_old = id_new; + id_new = transform_expr_once(scev, 0, id_old, priv, fn); + if (id_new < 0) + return id_new; + } while (id_old != id_new); + return id_new; +} + +static int simplify(struct scev *scev, u32 id, void *priv) +{ + u32 l, r, base, slope; + s64 imm1, imm2; + + /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */ + if (is_add(scev, id, &l, &r) && + is_linear(scev, l, &base, &slope) && + is_imm(scev, r, &imm1)) + return expr2(scev, LINEAR_SCEV, expr2(scev, BPF_ADD, base, r), slope); + + /* (+ imm imm) -> imm */ + if (is_add(scev, id, &l, &r) && + is_imm(scev, l, &imm1) && + is_imm(scev, r, &imm2)) + return imm_expr(scev, imm1 + imm2); + + /* (zext32 (imm X)) -> (imm (u32)X) */ + if (is_zext32(scev, id, &l) && is_imm(scev, l, &imm1)) + return imm_expr(scev, (u64)(u32)imm1); + + return id; +} + +static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) +{ + struct scev *scev = env->scev; + int l, r, id, ra, rb, ra_expr; + s64 imm; + + /* + * Note: + * reg2expr[ra] describes a change in rA's value over a single loop iteration. + * It does not account for rA's entry value. However, the SCEV constructed for rA + * must include an additional rA entry edge. For example: + * + * r0 = ..., r1 = 7 + * l: if r0 == 10 goto e // reg2expr[r1] == (imm 42) => reg2scev[r1] == (any r1 42) + * r0 += 1 + * r1 = 42 + * goto l + * e: + * ... at exit r1 is in {7,42} ... + */ + for (ra = 0; ra < REGS_NUM; ra++) { + ra_expr = transform_expr(scev, header_env->reg2expr[ra], NULL, simplify); + if (ra_expr < 0) + return ra_expr; + header_env->reg2expr[ra] = ra_expr; + /* rA = (+ rA IMM) -> (linear rA IMM) */ + if (is_add(scev, ra_expr, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = rA -> rA*/ + if (is_reg(scev, ra_expr, &rb) && ra == rb) { + header_env->reg2scev[ra] = ra_expr; + continue; + } + /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */ + if (is_any_imm_reg_opaque(scev, ra_expr)) { + id = mk_any(scev, expr1(scev, REG, ra), ra_expr); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + + } + return 0; +} +static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) +{ + u32 reg, *reg2scev = priv; + + /* (reg r) -> ... equation at header ... */ + if (is_reg(scev, id, ®)) + return reg2scev[reg]; + /* keep expression shapes */ + return id; +} + +static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn) +{ + struct scev *scev = env->scev; + int id, reg; + + for (reg = 0; reg < REGS_NUM; reg++) { + id = einsn->reg2expr[reg]; + id = transform_expr_once(scev, 0, id, eheader->reg2scev, instantiate_header_scevs); + if (id < 0) + return id; + id = transform_expr(scev, id, NULL, simplify); + if (id < 0) + return id; + einsn->reg2scev[reg] = id; + } + + return 0; +} + +static int log_scevs(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct env *header_env, *latch_env; + struct scev *scev = env->scev; + struct bpf_loop *loop; + int i, j, k, len, latch; + + len = env->prog->len; + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + header_env = find_header_env(scev, i); + bpf_log(log, "scev at header %d:\n", i); + print_env(env, header_env, i); + if (!can_compute_loop_scev(loop)) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + for (k = 0; k < loop->backedges[j].latches_cnt; k++) { + latch = loop->backedges[j].latches[k]; + bpf_log(log, " scev at %s %d:\n", k ? "exit" : "latch", latch); + latch_env = find_loop_env(scev, i, latch); + if (verifier_bug_if(!latch_env, env, "i=%d, latch=%d", i, latch)) + return -EFAULT; + print_env(env, latch_env, latch); + } + } + } + return 0; +} + +int bpf_compute_scev(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct scev *scev = env->scev; + int *postorder = env->cfg.insn_postorder; + int cnt = env->cfg.cur_postorder; + int i, idx, err, header; + + mark_latches(env); + /* + * Visit loop headers in postorder, to guarantee that scevs + * for innermost loops are computed first. + */ + for (i = 0; i < cnt; i++) { + idx = postorder[i]; + if (!aux[idx].loop) + continue; + err = compute_scev_for_loop(env, idx); + if (err) + return err; + } + + /* + * Compute scevs from exprs collected on a previous step. Iterate instructions in + * reverse post-order so that each loop header is processed before instructions + * reachable from it. + */ + for (i = cnt - 1; i >= 0; i--) { + idx = postorder[i]; + if (!aux[idx].need_scev) + continue; + header = bpf_loop_at_index(env, idx); + err = aux[idx].loop + ? compute_header_scevs(env, find_header_env(scev, header)) + : compute_insn_scevs(env, + find_header_env(scev, header), + find_loop_env(scev, header, idx)); + if (err) + return err; + } + + for (i = 0; i < env->prog->len; i++) + if (aux[i].loop) + aux[i].prune_point = true; + + if (env->log.level & BPF_LOG_LEVEL2) { + err = log_scevs(env); + if (err) + return err; + } + + return 0; +} + +void bpf_free_scev(struct bpf_verifier_env *env) +{ + struct scev *scev = env->scev; + struct insn_envs *envs; + int i; + u32 j; + + if (!scev) + return; + for (i = 0; i < scev->envs_cnt; i++) { + envs = scev->envs[i]; + if (!envs) + continue; + for (j = 0; j < envs->cnt; j++) + kfree(envs->entries[j].env); + kfree(envs); + } + if (scev->exprs_ht) + for (i = 0; i < scev->exprs_ht_cnt; i++) + kvfree(scev->exprs_ht[i]); + kvfree(scev->exprs_ht); + kvfree(scev->worklist.data); + kvfree(scev->envs); + kvfree(scev->exprs); + kvfree(scev->discovered); + kfree(scev); + env->scev = NULL; +} + +int bpf_init_scev(struct bpf_verifier_env *env) +{ + struct scev *scev; + + scev = kzalloc(sizeof(struct scev), GFP_KERNEL_ACCOUNT); + if (!scev) + return -ENOMEM; + env->scev = scev; + /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ + scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); + scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); + if (!scev->exprs_ht) + goto nomem; + if (expr0(scev, UNKNOWN) < 0 || expr0(scev, OPAQUE) < 0) + goto nomem; + scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); + scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); + if (!scev->envs || !scev->discovered) + goto nomem; + /* + * Remember original program length, in case bpf_free_scev() + * is called after bpf program rewrites that increase program + * length. + */ + scev->envs_cnt = env->prog->len; + return 0; +nomem: + bpf_free_scev(env); + return -ENOMEM; +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 85971f9c09c4..60287049cfe4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23233,6 +23233,14 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (env->log.level & BPF_LOG_LEVEL2) log_program(env); + ret = bpf_init_scev(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_scev(env); + if (ret < 0) + goto skip_full_check; + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; @@ -23382,6 +23390,7 @@ err_free_env: bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); + bpf_free_scev(env); bpf_stack_liveness_free(env); kvfree(env->cfg.postorder_nums); kvfree(env->cfg.insn_postorder); From 6ccf49f7932781f441e1e52459a4053bbfd19ddb Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:52 -0700 Subject: [PATCH 422/660] bpf: Use SCEV to widen bounded loops Loop-bound computation and register widening =========================================== During the main verification pass, use SCEV expressions and register values at the entry to the loop to estimate the number of iterations the loop might execute. Use this number to widen loop induction variables. For example: 0: r7 = 5; 1: r6 = 0; 2: while (r6 < 3) { // latch 3: if (r6 == 1) 4: r7 = 10; 5: r6 += 1; 6: } 8: use(r7); - Scalar evolution expressions (SCEVs) at the latch are: - r6 = (linear r6 1) - r7 = (any r7 10) - For this pre-condition loop, this corresponds to the linear inequality `r6_entry + n < 3`. With r6_entry = 0, the inequality is `n < 3`. - Hence infer that r6's range at line 2 is [0,3], including the final failed test. The body starts with r6 in [0,2]. - r7's (any r7 10) expression means that its initial value 5 from line 0 is joined with 10 from line 4: it is widened to [5,10] within and after the loop. Currently such inference is supported for: - Reducible loops with one backedge and complete backedge/exit lists. - A linear latch dominating the backedge, with one successor exiting the loop. Additional exits are allowed; they can shorten execution. - 64-bit arithmetic and comparisons whose continuation condition normalizes to signed/unsigned < or <=, or !=. - Initial counter, latch offset, slope and bound evaluable as constants at loop entry, yielding a finite, representable iteration bound. - Values with an eligible type at loop entry, e.g. SCALAR_VALUE or PTR_TO_STACK, but not PTR_TO_CTX. In order for widening to proceed, every register live at the loop entry has to have a SCEV that can be widened. Currently, the following expressions are supported: - (linear ): entry value + iteration# * constant step. - (reg ): loop invariant; keep its entry value unchanged. - (any ...): join the entry value with immediates or invariant registers; types must match, and immediates require SCALAR_VALUE. Main verification loop integration ================================== Before proceeding with the usual instruction processing, do_check() takes the following steps at loop entries (H): - Checks whether the loop can be widened and obtains its iteration bound (scev.c:bpf_compute_loop_iters()). - If it can, saves an unwidened checkpoint E0, which will be used as a base for deriving widened values throughout this invocation of the loop. - Widens registers in the current state using E0 and SCEV expressions computed for H (scev.c:bpf_widen_scev_regs()). - Pushes a per-frame loop-stack record to the current verifier state. For terminating loops, the record stores E0 and the iteration bounds. - Lets is_state_visited() create a new checkpoint W at the same header, now containing the widened values, and proceeds with normal checking. - On an exit edge, pops the loop record and verifies the exit path normally. is_state_visited() treats a loop proven to terminate like an iterator-based loop: RANGE_WITHIN comparison attempts to establish convergence against a checkpoint whose exploration is still in progress. For example: r0 = 0; H: r0 += 1; if (r0 != 3) goto H; exit; first H: prove H_count = 3; save E0(r0 = 0) widen r0 to [0,2]; push the loop record save W(r0 = [0,2]) body: r0 becomes [1,3]; fork at the conditional exit: r0 = 3; pop the loop record and check exit backedge: r0 = [1,2]; clamp, then match with W and prune Clamping -------- Independent register ranges lose correlations between induction variables. For example: r6 = 0; r7 = 0; do { r6 += 1; r7 += 2; } while (r6 < 3); use(r7); header: r6 in [0,2], r7 in {0,2,4} body: r6 in [1,3], r7 in {2,4,6} backedge: r6 in [1,2], r7 still in {2,4,6} The latch narrows r6 but does not propagate this refinement to r7. To account for this, before comparing states on a backedge, scev.c:bpf_clamp_scev_regs() recomputes r7's range from E0 and the saved iteration bound, then intersects the current range {2,4,6} with the recomputed {0,2,4}, arriving at {2,4}. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-34-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 33 + kernel/bpf/log.c | 5 + kernel/bpf/scev.c | 1039 +++++++++++++++++ kernel/bpf/states.c | 88 +- kernel/bpf/verifier.c | 205 +++- .../selftests/bpf/progs/verifier_gotox.c | 6 +- 6 files changed, 1364 insertions(+), 12 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 52f72e92eb0c..d760d2283ac5 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -328,6 +328,25 @@ struct bpf_retval_range { bool return_32bit; }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + +struct bpf_loop_iters { + u32 min_header_count; /* min number of times header is executed */ + u32 max_header_count; /* max number of times header is executed */ + unsigned long precise_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + +struct loop_stack_entry { + struct bpf_verifier_state *entry_state; + u32 min_header_count; + u32 max_header_count; + u32 loop_id:31; + u32 terminates:1; +}; + +#define LOOP_STACK_SIZE 16 + /* state of the program: * type of all registers and stack info */ @@ -373,6 +392,11 @@ struct bpf_func_state { u32 callback_depth; /* Instructions processed in this frame and callees on the current path. */ u32 insns_subtotal; + /* + * Control-flow loop nesting at the current insn within this frame's + * subprogram (loops never cross subprogram boundaries). + */ + u32 loop_stack_cnt; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -390,6 +414,7 @@ struct bpf_func_state { u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ + struct loop_stack_entry *loop_stack; }; #define MAX_CALL_FRAMES 16 @@ -436,6 +461,7 @@ static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) #define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ MAX_CALL_FRAMES) + struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -1971,4 +1997,11 @@ int bpf_init_scev(struct bpf_verifier_env *env); void bpf_free_scev(struct bpf_verifier_env *env); int bpf_compute_scev(struct bpf_verifier_env *env); +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters); +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters); +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 8ee7d7b71147..5432c0fc442b 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -792,6 +792,11 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, " cb"); if (state->in_async_callback_fn) verbose(env, " async_cb"); + if (state->loop_stack_cnt) { + verbose(env, " loop_stack="); + for (i = 0; i < state->loop_stack_cnt; i++) + verbose(env, "%s%d", i ? "," : "", state->loop_stack[i].loop_id); + } verbose(env, "\n"); if (!print_all) mark_verifier_state_clean(env); diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 2d2a84156180..05dd8e439609 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -1,11 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ +#include "linux/cnum.h" #include #include #include #include #include +#include +#include #define REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) #define UNKNOWN_EXPR_ID 0 @@ -80,6 +83,9 @@ struct expr_stack_elt { DEFINE_MIN_HEAP(int, scev_worklist); struct scev { + /* Read-only representatives for stack slots without a saved register. */ + struct bpf_reg_state zero_reg; + struct bpf_reg_state unknown_reg; /* * Expressions are identified by id, exprs_ht ensures that * each expression exists as a unique instance. @@ -1616,6 +1622,8 @@ int bpf_init_scev(struct bpf_verifier_env *env) if (!scev) return -ENOMEM; env->scev = scev; + bpf_mark_reg_known_scalar(&scev->zero_reg, 0); + bpf_mark_reg_unknown_imprecise(&scev->unknown_reg); /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); @@ -1638,3 +1646,1034 @@ nomem: bpf_free_scev(env); return -ENOMEM; } + +static struct bpf_stack_state *scev_regno_to_stack(const struct bpf_func_state *st, u32 r) +{ + int spi, slots_available; + + if (r < MAX_BPF_REG) + return NULL; + + slots_available = st->allocated_stack / BPF_REG_SIZE; + spi = r - MAX_BPF_REG; + if (spi < slots_available) + return &st->stack[spi]; + + return NULL; +} + +static struct bpf_reg_state *scev_regno_to_reg(struct bpf_func_state *st, u32 r) +{ + return r < MAX_BPF_REG ? &st->regs[r] : NULL; +} + +/* Linear widening requires a physical register or a full spill. */ +static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 r) +{ + struct bpf_stack_state *stack; + + if (r < MAX_BPF_REG) + return scev_regno_to_reg(st, r); + stack = scev_regno_to_stack(st, r); + if (!stack || bpf_spill_size(stack) != BPF_REG_SIZE) + return NULL; + return &stack->spilled_ptr; +} + +/* + * Return a borrowed value; never use it as a widening destination. + * NULL denotes no readable value. + */ +static const struct bpf_reg_state * +scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, u32 r) +{ + const struct bpf_stack_state *stack; + bool zero = true; + int i; + + if (r < MAX_BPF_REG) + return st->regs[r].type == NOT_INIT ? NULL : &st->regs[r]; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return NULL; + + stack = scev_regno_to_stack(st, r); + if (!stack) + return &env->scev->unknown_reg; + if (bpf_spill_size(stack) == BPF_REG_SIZE) + return &stack->spilled_ptr; + + /* Partial spills do not describe the value of the whole slot. */ + for (i = 0; i < BPF_REG_SIZE; i++) { + switch (stack->slot_type[i]) { + case STACK_ZERO: + break; + case STACK_INVALID: + case STACK_POISON: + case STACK_MISC: + zero = false; + break; + case STACK_SPILL: + if (!bpf_is_spilled_scalar_reg(stack)) + return NULL; + zero = false; + break; + default: + return NULL; + } + } + return zero ? &env->scev->zero_reg : &env->scev->unknown_reg; +} + +/* Materialize a writable value in the current state, growing its stack if needed. */ +static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st, + u32 r, struct bpf_reg_state **out) +{ + const struct bpf_reg_state *value; + struct bpf_stack_state *stack; + u32 spi; + int err; + + *out = scev_regno_to_reg(st, r); + if (*out) + return 0; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return -EINVAL; + + spi = r - MAX_BPF_REG; + err = bpf_grow_stack_state(env, st, (spi + 1) * BPF_REG_SIZE); + if (err) + return err; + + /* Resolve pointers after growth, which may relocate the stack. */ + value = scev_reg_for_read(env, st, r); + if (!value) + return -EINVAL; + stack = scev_regno_to_stack(st, r); + stack->spilled_ptr = *value; + memset(stack->slot_type, STACK_SPILL, sizeof(stack->slot_type)); + *out = &stack->spilled_ptr; + return 0; +} + +static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 r) +{ + int insn_idx = bpf_frame_insn_idx(st, st->curframe); + u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before; + int spi; + + if (r < MAX_BPF_REG) { + return BIT(r) & live_regs; + } else { + spi = r - MAX_BPF_REG; + return bpf_stack_slot_alive(env, st->curframe, spi * 2) || + bpf_stack_slot_alive(env, st->curframe, spi * 2 + 1); + } +} + +/* + * Latch is a condition deciding if execution remains inside a loop. + * Linear latch represents a condition 'if goto ', + * where equation ' + i * ' describes values taken by register , + * 'i' is the loop iteration number, starting from 0. + */ +struct linear_latch { + u32 insn_idx; + u32 base_expr; + u32 step_expr; + u32 bound_expr; + u32 op; +}; + +static bool loop_invariant(struct scev *scev, u32 id) +{ + s64 imm; + u32 reg; + + return is_reg(scev, id, ®) || is_imm(scev, id, &imm); +} + +static int match_linear_latch(struct bpf_verifier_env *env, + u32 header, + u32 latch_idx, + struct linear_latch *latch) +{ + struct bpf_insn *insn = &env->prog->insnsi[latch_idx]; + struct scev *scev = env->scev; + struct env *latch_env; + u32 true_branch_loop; + u32 true_branch_tgt; + u32 src_reg_scev; + u32 dst_reg_scev; + u32 t, l, r, op; + int id; + + /* 32-bit arithmetic is not handled yet */ + if (BPF_CLASS(insn->code) != BPF_JMP) + return false; + op = BPF_OP(insn->code); + /* Flip the condition if true branch jumps out of the loop. */ + true_branch_tgt = latch_idx + bpf_jmp_offset(insn) + 1; + true_branch_loop = bpf_loop_at_index(env, true_branch_tgt); + if (true_branch_loop != header && + !bpf_is_nested_loop(env, true_branch_loop, header)) + op = bpf_rev_opcode(op); + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + case BPF_JLT: + case BPF_JLE: + case BPF_JGT: + case BPF_JGE: + case BPF_JNE: + break; + default: + return false; + } + + latch->insn_idx = latch_idx; + + latch_env = find_loop_env(scev, header, latch_idx); + if (verifier_bug_if(!latch_env, env, "header_idx=%d, latch_idx=%d\n", header, latch_idx)) + return -EFAULT; + dst_reg_scev = latch_env->reg2scev[insn->dst_reg]; + src_reg_scev = latch_env->reg2scev[insn->src_reg]; + /* + * Make sure (linear ...) is in dst_reg_scev: + * 'if ' ⇔ + * 'if '. + */ + if (BPF_SRC(insn->code) == BPF_X && is_linear(scev, src_reg_scev, &l, &r)) { + swap(dst_reg_scev, src_reg_scev); + op = bpf_flip_opcode(op); + } + latch->op = op; + /* + * simplify() produces two shapes for linear latches: + * - (linear register step) + * ^----------------------------+-- base_expr + * base_expr | + * - (linear (+ register imm) step) | + * ^----------------------------' + */ + if (!is_linear(scev, dst_reg_scev, &latch->base_expr, &latch->step_expr)) + return false; + + if (BPF_SRC(insn->code) == BPF_K) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + latch->bound_expr = id; + } else { + latch->bound_expr = src_reg_scev; + } + + if (!loop_invariant(scev, latch->step_expr) || + !loop_invariant(scev, latch->bound_expr)) + return false; + + /* (linear register step) */ + if (is_reg(scev, latch->base_expr, &t)) + return true; + + /* (linear (+ register imm) step) */ + if (is_add(scev, latch->base_expr, &l, &r) && + is_reg(scev, l, &t) && + loop_invariant(scev, r)) + return true; + + return false; +} + +struct scev_value { + u64 value; /* scalar value or a pointer offset */ + int ptr_reg; /* SCEV register identifying the pointer origin, or -1. */ +}; + +static void log_scev_value(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *v) +{ + const struct bpf_reg_state *reg; + + if (v->ptr_reg >= 0) { + reg = scev_reg_for_read(env, st, v->ptr_reg); + log_reg(env, v->ptr_reg); + bpf_log(&env->log, " (%s)", reg ? reg_type_str(env, reg->type) : "unavailable"); + } else { + bpf_log(&env->log, "scalar %llu", v->value); + } +} + +static void log_incompatible_scev_values(struct bpf_verifier_env *env, const char *reason, + struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + bpf_log(&env->log, "scev: %s ", reason); + log_scev_value(env, st, a); + bpf_log(&env->log, ", "); + log_scev_value(env, st, b); + bpf_log(&env->log, "\n"); +} + +static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + const struct bpf_reg_state *ra, *rb; + + if (a->ptr_reg < 0 || b->ptr_reg < 0) { + if (a->ptr_reg < 0 && b->ptr_reg < 0) + return true; + goto incompatible; + } + ra = scev_reg_for_read(env, st, a->ptr_reg); + rb = scev_reg_for_read(env, st, b->ptr_reg); + if (ra && rb && bpf_same_memory_origin(ra, rb)) + return true; + +incompatible: + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "incompatible latch operands", st, a, b); + return false; +} + +static bool scev_value_add(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b, + struct scev_value *result) +{ + if (a->ptr_reg >= 0 && b->ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "can't add two pointers", st, a, b); + return false; + } + result->value = a->value + b->value; + result->ptr_reg = a->ptr_reg >= 0 ? a->ptr_reg : b->ptr_reg; + return true; +} + +static bool eval_expr(struct bpf_verifier_env *env, struct scev *scev, + struct bpf_func_state *st, u32 id, struct scev_value *result) +{ + struct scev_value lval, rval; + const struct bpf_reg_state *reg; + u32 l, r, regno; + s64 imm; + + if (is_reg(scev, id, ®no)) { + reg = scev_reg_for_read(env, st, regno); + if (!reg || reg->type == NOT_INIT || !tnum_is_const(reg->var_off)) + return false; + result->value = reg->var_off.value; + result->ptr_reg = reg->type == SCALAR_VALUE ? -1 : (int)regno; + return true; + } else if (is_imm(scev, id, &imm)) { + result->value = imm; + result->ptr_reg = -1; + return true; + } else if (is_add(scev, id, &l, &r) && + eval_expr(env, scev, st, l, &lval) && + eval_expr(env, scev, st, r, &rval)) { + return scev_value_add(env, st, &lval, &rval, result); + } + return false; +} + +/* + * Loop with post-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2] header executed 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * if r0 != 3 goto l r0 ∈ [1,2,3] backedge taken 2 times + * ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = 1 + k + * + * Loop with pre-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2,3] header executed 4 times + * if r0 == 3 goto e r0 ∈ [0,1,2,3] backedge taken 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * goto l r0 ∈ [1,2,3] + * e: ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = k + */ +static bool compute_max_iters(struct bpf_verifier_env *env, + struct bpf_func_state *st, + struct linear_latch *latch, + struct bpf_loop_iters *iters) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop; + struct scev *scev = env->scev; + struct scev_value initial_val, step_val, bound_val; + u64 step, diff, bound, initial, backedge_taken_count; + u8 op = latch->op; + bool up, down; + + if (!eval_expr(env, scev, st, latch->base_expr, &initial_val) || + !eval_expr(env, scev, st, latch->step_expr, &step_val) || + !eval_expr(env, scev, st, latch->bound_expr, &bound_val)) + return false; + + if (step_val.ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) { + bpf_log(&env->log, "scev: latch step is a pointer "); + log_scev_value(env, st, &step_val); + bpf_log(&env->log, "\n"); + } + return false; + } + if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + return false; + + initial = initial_val.value; + step = step_val.value; + bound = bound_val.value; + + if (step == 0) + return false; + down = (s64)step < 0; + up = (s64)step > 0; + diff = bound - initial; + switch (op) { + case BPF_JLT: + /* + * E.g.: 3 + 2*i < 7 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. + * Here the inequality describes a set of values `i` might take + * while the latch condition remains true (the backedge is taken). + * The backedge_taken_count is the size of this set. + */ + if (/* + * If the condition is initially true, decreasing values won't + * change that without underflow. If the condition is initially false, + * the main pass would do a single iteration anyway. + */ + down || + /* Is the latch condition false on the first iteration? */ + initial >= bound || + /* Count iterations satisfying the strict inequality, including i=0. */ + check_add_overflow((diff - 1) / step, 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the counter: + * initial + backedge_taken_count * step > U64_MAX + */ + backedge_taken_count > (U64_MAX - initial) / step) + return false; + break; + case BPF_JLE: + /* E.g.: 3 + 2*i ≤ 7 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (down || initial > bound || + /* Count iterations satisfying the non-strict inequality. */ + check_add_overflow(diff / step, 1, &backedge_taken_count) || + backedge_taken_count > (U64_MAX - initial) / step) + return false; + break; + case BPF_JGT: + /* E.g.: 7 - 2*i > 3 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. */ + if (up || initial <= bound || + check_add_overflow((-diff - 1) / -step, 1, &backedge_taken_count) || + /* + * The last iteration underflows the counter if: + * initial + backedge_taken_count * step < 0 ⇔ + * backedge_taken_count * step < -initial ⇔ + * backedge_taken_count > initial / -step + * (direction flips because step < 0) + */ + backedge_taken_count > initial / -step) + return false; + break; + case BPF_JGE: + /* E.g.: 7 - 2*i ≥ 3 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (up || initial < bound || + check_add_overflow(-diff / -step, 1, &backedge_taken_count) || + backedge_taken_count > initial / -step) + return false; + break; + case BPF_JSLT: + if (down || (s64)initial >= (s64)bound || + check_add_overflow((diff - 1) / step, 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the signed counter: + * (s64)initial + backedge_taken_count * step > S64_MAX + */ + backedge_taken_count > ((u64)S64_MAX - initial) / step) + return false; + break; + case BPF_JSLE: + if (down || (s64)initial > (s64)bound || + check_add_overflow(diff / step, 1, &backedge_taken_count) || + backedge_taken_count > ((u64)S64_MAX - initial) / step) + return false; + break; + case BPF_JSGT: + if (up || (s64)initial <= (s64)bound || + check_add_overflow((-diff - 1) / -step, 1, &backedge_taken_count) || + /* + * The last iteration underflows the signed counter if: + * (s64)initial + backedge_taken_count * step < S64_MIN ⇔ + * backedge_taken_count * step < S64_MIN - (s64)initial ⇔ + * backedge_taken_count > initial - (u64)S64_MIN / -step + * (direction flips because step < 0) + */ + backedge_taken_count > (initial - (u64)S64_MIN) / -step) + return false; + break; + case BPF_JSGE: + if (up || (s64)initial < (s64)bound || + check_add_overflow(-diff / -step, 1, &backedge_taken_count) || + backedge_taken_count > (initial - (u64)S64_MIN) / -step) + return false; + break; + case BPF_JNE: + /* + * For a decreasing counter flip the diff and step: + * 7 - 2*i ≠ 3 ⇔ 2*i ≠ 7 - 3. + */ + diff = down ? -diff : diff; + step = down ? -step : step; + if (diff == 0 || diff % step) + return false; + backedge_taken_count = diff / step; + break; + default: + return false; + } + + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + + /* The header executes once on entry and once for each taken backedge, hence +1. */ + iters->max_header_count = backedge_taken_count + 1; + /* + * The latch is a conditional jump with one jump target exiting the loop. + * Linear latch is matched only if the loop has a single backedge. + * The loop still, however can have multiple exits. + * In such case, conservatively assume that non-latch exit can happen + * at any iteration, thus setting minimal number of iterations as 0. + */ + iters->min_header_count = loop->exits_cnt == 1 ? iters->max_header_count : 0; + return true; +} + +static void mark_scev_reg_scratched(struct bpf_verifier_env *env, u32 r) +{ + if (r < MAX_BPF_REG) + mark_reg_scratched(env, r); + else + mark_stack_slot_scratched(env, r - __MAX_BPF_REG); +} + +/* + * An unknown scalar is its own widening. + * Stack bytes without a value are represented by scev->unknown_reg. + */ +static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_func_state *st, + u32 r) +{ + const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r); + + return reg && reg->type == SCALAR_VALUE && !reg->id && reg->step <= 1 && + tnum_is_unknown(reg->var_off) && + reg->r64.size == U64_MAX && reg->r32.size == U32_MAX; +} + +/* Main logic in verifier.c forbids varying offsets for certain register types. */ +static bool is_widenable_reg_type(const struct bpf_reg_state *reg) +{ + if (type_may_be_null(reg->type)) + return false; + + switch (base_type(reg->type)) { + case SCALAR_VALUE: + case PTR_TO_MAP_VALUE: + case PTR_TO_MAP_KEY: + case PTR_TO_STACK: + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + case PTR_TO_MEM: + case PTR_TO_BUF: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) +{ + switch (base_type(reg->type)) { + case SCALAR_VALUE: + reg->id = 0; + break; + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + reg->id = ++env->id_gen; + break; + default: + break; + } +} + +/* Check that ANY leaves can be unioned with r's loop-entry value. */ +static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct env *header_env, u32 r, u32 id) +{ + const struct bpf_reg_state *reg, *leaf_reg; + struct scev *scev = env->scev; + u32 l, ra, rr, leaf, order; + s64 imm; + + if (!is_any(scev, id, &l, &rr)) + return false; + reg = scev_reg_for_read(env, loop_entry, r); + if (!reg || !is_widenable_reg_type(reg)) + return false; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || is_any(scev, id, &l, &rr)) + continue; + if (is_imm(scev, id, &imm)) { + if (reg->type != SCALAR_VALUE) + return false; + } else if (is_reg(scev, id, &leaf)) { + /* Other leaves must denote loop-invariant registers. */ + if (leaf != r && + !(is_reg(scev, header_env->reg2scev[leaf], &ra) && ra == leaf)) + return false; + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (!leaf_reg || !bpf_reg_union_compatible(reg, leaf_reg)) + return false; + } else { + return false; + } + } + return true; +} + +struct bounds { + struct cnum64 range; + u16 base; + u16 step; +}; + +static bool is_simple_linear(struct scev *scev, u32 id, u32 *base_reg, s64 *slope_imm) +{ + u32 base, slope; + + return is_linear(scev, id, &base, &slope) && + is_reg(scev, base, base_reg) && + is_imm(scev, slope, slope_imm) && + *slope_imm <= S16_MAX && + *slope_imm >= S16_MIN && + *slope_imm != 0; +} + +/* + * Compute the range an induction variable in `reg` spans over the loop. + * If the computation overflows s64 the range is unbounded, only the + * power-of-two alignment survives the wraparound. + */ +static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, s64 slope, + struct bounds *out) +{ + s64 slope_abs = slope < 0 ? -slope : slope; + s64 min_val = reg_smin(reg); + s64 max_val = reg_smax(reg); + s64 total_change; + bool overflow; + + overflow = check_mul_overflow(slope, (s64)max_header_count - 1, &total_change) || + (slope > 0 && check_add_overflow(max_val, total_change, &max_val)) || + (slope < 0 && check_add_overflow(min_val, total_change, &min_val)); + /* + * If the entry value is a single point the value set is 'v + slope * k', + * so the step is |slope|. Otherwise, only the power-of-two alignment + * shared by the entry value and the slope. + */ + if (cnum64_is_const(reg->r64) && !overflow) { + out->step = slope_abs; + out->base = imod(min_val, out->step); + } else { + out->step = 1u << min_t(u32, tnum_alignment(reg->var_off), __ffs(slope_abs)); + out->base = 0; + } + out->range = overflow ? CNUM64_UNBOUNDED : cnum64_from_srange(min_val, max_val); +} + +static int mark_expr_precise(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 id, + unsigned long *precise_regs) +{ + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + u32 r, order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev precision marking exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || !is_reg(scev, id, &r)) + continue; + /* Synthetic values have no spilled register to mark precise. */ + reg = scev_reg_for_linear(st, r); + if (reg && reg->type == SCALAR_VALUE) + __set_bit(r, precise_regs); + } + return 0; +} + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct env *header_env; + struct linear_latch latch, best; + struct bpf_backedge *backedge; + struct bpf_loop_iters tmp; + struct bpf_loop *loop; + int insn_idx = st->insn_idx; + bool found = false; + int linear_latch; + u32 r, base_reg; + int latch_idx; + s64 slope_imm; + int err, j; + + if (!env->allow_uninit_stack) + return 0; + + /* + * If insn_idx is a loop header for a reducible loop with a single backedge. + * loop is NULL for secondary entries to irreducible loops. + */ + loop = aux[insn_idx].loop; + if (!loop || loop->irreducible || loop->backedges_cnt != 1 || loop->backedges_overflow || + loop->exits_overflow) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, unsupported loop:%s%s%s\n", insn_idx, + !loop || loop->irreducible ? " irreducible" : "", + loop && loop->backedges_cnt > 1 ? " multiple backedges" : "", + loop && loop->exits_overflow ? " too many exits" : ""); + return 0; + } + + /* If this backedge has a latch */ + backedge = &loop->backedges[0]; + if (!backedge->latches_cnt) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, latch not identified\n", insn_idx); + return 0; + } + + /* + * Every condition dominating the backedge bounds the number of + * iterations from above, keep the smallest bound. + */ + for (j = 0; j < backedge->latches_cnt; j++) { + latch_idx = backedge->latches[j]; + linear_latch = match_linear_latch(env, insn_idx, latch_idx, &latch); + if (linear_latch < 0) + return linear_latch; + if (!linear_latch) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, non-linear %s at %d\n", + insn_idx, j ? "exit" : "latch", latch_idx); + continue; + } + if (!compute_max_iters(env, cur_func, &latch, &tmp)) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, + "loop header at %d, can't compute iterations count at %d\n", + insn_idx, latch_idx); + continue; + } + if (!found || tmp.max_header_count < iters->max_header_count) { + iters->max_header_count = tmp.max_header_count; + best = latch; + } + iters->min_header_count = found ? min(iters->min_header_count, tmp.min_header_count) + : tmp.min_header_count; + found = true; + } + if (!found) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, can't compute iterations count\n", insn_idx); + return 0; + } + latch = best; + + if (iters->max_header_count == 0) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, 0 iterations count\n", insn_idx); + return 0; + } + + if (iters->max_header_count == U32_MAX) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, inf iterations count\n", insn_idx); + return 0; + } + + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, header_count is ", insn_idx); + if (iters->min_header_count == iters->max_header_count) + bpf_log(log, "%u ", iters->max_header_count); + else + bpf_log(log, "[%u..%u] ", iters->min_header_count, iters->max_header_count); + bpf_log(log, "\n"); + } + + err = bpf_live_stack_query_init(env, st); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + struct bpf_reg_state *reg; + u32 ra, r_scev, r_expr; + + if (!scev_reg_alive(env, st, r)) + continue; + /* + * A heuristic: + * Ignore SCEVs computed for unbound values and treat those as widened. + * This is poor man's modelling of e.g. bpf_probe_read_kernel() effect + * on stack, when called from a subprogram. Currently this would produce + * an unknown ('?') SCEV as a summary of a subprogram call effects. + * Ideally, this should be computed as an OPAQUE value. + * However, that would require tracking STACK_MISC writes in the + * liveness analysis. Not implemented yet. + */ + if (is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_scev = header_env->reg2scev[r]; + reg = scev_reg_for_linear(cur_func, r); + /* If SCEV for r is (linear ) */ + if (is_simple_linear(scev, r_scev, &base_reg, &slope_imm) && + reg && is_widenable_reg_type(reg)) { + continue; + } + + /* rA = rA, loop does not change this reg */ + if (is_reg(scev, r_scev, &ra) && r == ra) + continue; + + /* (any 1 (any 2 (any 3 4))) */ + if (is_widenable_any(env, cur_func, header_env, r, r_scev)) { + continue; + } + + if (log->level & BPF_LOG_LEVEL2) { + r_expr = header_env->reg2expr[r]; + bpf_log(log, "loop header at %d, can't widen ", insn_idx); + log_reg(env, r); + bpf_log(log, ", expr is "); + log_expr(env, r_expr); + bpf_log(log, "\n"); + } + return 0; + } + + bitmap_zero(iters->precise_regs, REGS_NUM); + err = mark_expr_precise(env, cur_func, latch.base_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.step_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.bound_expr, iters->precise_regs); + if (err) + return err; + + return 1; +} + +/* bpf_compute_loop_iters() checked that every leaf can be unioned into acc. */ +static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct bpf_reg_state *acc, u32 id) +{ + struct bpf_reg_state *tmp = &env->fake_reg[0]; + const struct bpf_reg_state *leaf_reg; + struct scev *scev = env->scev; + u32 l, r, leaf, order; + s64 imm; + int err; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev ANY union exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || is_any(scev, id, &l, &r)) + continue; + if (is_imm(scev, id, &imm)) { + bpf_mark_reg_known_scalar(tmp, imm); + leaf_reg = tmp; + } else if (is_reg(scev, id, &leaf)) { + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (verifier_bug_if(!leaf_reg, env, "unreadable SCEV ANY leaf %u", leaf)) + return -EFAULT; + } else { + verifier_bug(env, "scev ANY union has an unsupported leaf"); + return -EFAULT; + } + err = bpf_reg_union(env, acc, leaf_reg); + if (err) + return err; + } + return 0; +} + +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_func_state *entry_func = loop_entry->frame[loop_entry->curframe]; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + struct env *header_env; + struct bounds bounds; + u32 r, base_reg, r_expr, a, b; + int insn_idx = st->insn_idx; + s64 slope_imm; + int err; + + for_each_set_bit(r, iters->precise_regs, REGS_NUM) { + if (r < MAX_BPF_REG) + bpf_bt_set_frame_reg(&env->bt, st->curframe, r); + else + bpf_bt_set_frame_slot(&env->bt, st->curframe, r - MAX_BPF_REG); + } + err = bpf_mark_chain_precision(env, st, -1, NULL); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + if (!scev_reg_alive(env, st, r) || is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_expr = header_env->reg2scev[r]; + /* If SCEV for r is (linear )*/ + if (is_simple_linear(scev, r_expr, &base_reg, &slope_imm) && + base_reg == r) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + linear_bounds(reg, iters->max_header_count, slope_imm, &bounds); + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, cnum64_smin(bounds.range)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum64_smax(bounds.range)); + bpf_log(log, " step %u\n", bounds.step); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.range, bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } else if (is_any(scev, r_expr, &a, &b)) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + err = union_any_reg(env, entry_func, reg, r_expr); + if (err) + return err; + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, reg_smin(reg)); + bpf_log(log, ".."); + bpf_verbose_snum(env, reg_smax(reg)); + bpf_log(log, " step %u\n", reg->step); + } + scratch_widened_reg_id(env, reg); + mark_scev_reg_scratched(env, r); + } + } + return 1; +} + +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur_func_state, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count) +{ + struct bpf_func_state *entry_st = entry_state->frame[entry_state->curframe]; + struct bpf_verifier_log *log = &env->log; + struct bpf_reg_state *reg, *entry_reg; + struct scev *scev = env->scev; + struct env *header_env; + struct bounds bounds; + u32 r, base_reg; + s64 slope_imm; + int err; + + if (entry_state->curframe != cur_func_state->frameno) { + verifier_bug(env, "clamping registers for a wrong frame: %d vs %d\n", + entry_state->curframe, cur_func_state->frameno); + return -EFAULT; + } + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + /* If SCEV for r is (linear )*/ + if (!is_simple_linear(scev, header_env->reg2scev[r], &base_reg, &slope_imm) || + base_reg != r) + continue; + + entry_reg = scev_reg_for_linear(entry_st, r); + reg = scev_reg_for_linear(cur_func_state, r); + if (!entry_reg || !reg || entry_reg->type == NOT_INIT || reg->type == NOT_INIT || + is_unbound_scev_reg(env, entry_st, r)) + continue; + + linear_bounds(entry_reg, max_header_count, slope_imm, &bounds); + bounds.range = cnum64_intersect(reg->r64, bounds.range); + if (cnum64_is_empty(bounds.range)) { + verifier_bug(env, "scev clamp produced empty range for r%d", r); + return -EFAULT; + } + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, clamping ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, cnum64_smin(bounds.range)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum64_smax(bounds.range)); + bpf_log(log, " step %u\n", bounds.step); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.range, bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } + return 0; +} diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 470bff8d8dc8..c62ea992967b 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -936,6 +936,27 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c return true; } +static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state *old, + struct bpf_func_state *cur) +{ + struct loop_stack_entry *old_stack = old->loop_stack; + struct loop_stack_entry *cur_stack = cur->loop_stack; + u32 i; + + if (old->loop_stack_cnt != cur->loop_stack_cnt) + return false; + + for (i = 0; i < old->loop_stack_cnt; i++) { + if (old_stack[i].loop_id == cur_stack[i].loop_id && + old_stack[i].max_header_count == cur_stack[i].max_header_count && + old_stack[i].terminates == cur_stack[i].terminates) + continue; + return false; + } + + return true; +} + /* compare two verifier states * * all states stored in state_list are known to be valid, since @@ -986,6 +1007,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact)) return false; + if (!loop_stack_safe(env, old, cur)) + return false; + return true; } @@ -1032,6 +1056,7 @@ static bool states_equal(struct bpf_verifier_env *env, if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact)) return false; } + return true; } @@ -1314,15 +1339,40 @@ int bpf_split_cur_state(struct bpf_verifier_env *env) return 0; } +/* Force a checkpoint upon reaching a loop header for a terminating loop. */ +static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur->frame[cur->curframe]; + struct loop_stack_entry *top; + + if (bpf_loop_at_index(env, insn_idx) != insn_idx || !frame->loop_stack_cnt) + return false; + top = &frame->loop_stack[frame->loop_stack_cnt - 1]; + return top->loop_id == insn_idx && top->terminates; +} + +static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame = st->frame[st->curframe]; + + if (frame->loop_stack_cnt == 0) + return NULL; + return &frame->loop_stack[frame->loop_stack_cnt - 1]; +} + int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) { + struct loop_stack_entry *old_loop, *cur_loop; struct bpf_verifier_state_list *sl; struct bpf_verifier_state *cur = env->cur_state; - bool force_new_state, add_new_state, loop; + bool force_new_state, add_new_state, loop, loop_checkpoint; int n, err, states_cnt = 0; struct list_head *pos, *tmp, *head; + loop_checkpoint = need_loop_checkpoint(env, insn_idx); force_new_state = env->test_state_freq || bpf_is_force_checkpoint(env, insn_idx) || + loop_checkpoint || /* Avoid accumulating infinitely long jmp history */ cur->jmp_history_cnt > 40; @@ -1353,10 +1403,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) continue; if (sl->state.branches) { - struct bpf_func_state *frame = sl->state.frame[0]; + struct bpf_func_state *old_top_frame = sl->state.frame[0]; - if (frame->in_async_callback_fn && - frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { + if (old_top_frame->in_async_callback_fn && + old_top_frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { /* Different async_entry_cnt means that the verifier is * processing another entry into async callback. * Seeing the same state is not an indication of infinite @@ -1455,6 +1505,33 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) } goto skip_inf_loop_check; } + /* + * If old state belongs to a control flow loop that we know terminates, + * it should be safe to prune current state. However, account for the + * following situation: + * + * for (;;) { + * for (i = 0; i < 10; i++) + * ... + * } + * + * Here a backedge to a non-terminating outer loop is dominated by an exit + * from the inner loop. Pruning at a re-entry to the inner loop therefore + * would hide the fact that outer loop is non-terminating. + * Hence, only allow pruning states within the same entry state. + */ + old_loop = current_loop(&sl->state); + cur_loop = current_loop(cur); + if (old_loop && cur_loop && + old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state) { + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; + goto hit; + } + goto skip_inf_loop_check; + } + /* attempt to detect infinite loop to avoid unnecessary doomed work */ if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur, EXACT) && @@ -1613,7 +1690,8 @@ miss: * Use bigger 'n' for checkpoints because evicting checkpoint states * too early would hinder iterator convergence. */ - n = bpf_is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; + n = (bpf_is_force_checkpoint(env, insn_idx) || loop_checkpoint) && + sl->state.branches > 0 ? 64 : 3; if (sl->miss_cnt > sl->hit_cnt * n + n) { /* the state is unlikely to be useful. Remove it to * speed up verification diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 60287049cfe4..a597a8c18972 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1681,6 +1681,7 @@ static void free_func_state(struct bpf_func_state *state) { if (!state) return; + kfree(state->loop_stack); kfree(state->stack_arg_regs); kfree(state->stack); kfree(state); @@ -1717,6 +1718,10 @@ static int copy_func_state(struct bpf_func_state *dst, memcpy(dst, src, offsetof(struct bpf_func_state, stack)); /* Instruction accounting is path-local, not part of verifier state. */ dst->insns_subtotal = 0; + dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt, + sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT); + if (!dst->loop_stack) + return -ENOMEM; return copy_stack_state(dst, src); } @@ -19916,6 +19921,178 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return -EFAULT; } +/* + * Push the loop entered at insn_idx onto the stack. Usually this adds a single + * entry on top of its enclosing loop. However, an edge may enter an inner loop + * directly, bypassing the header(s) of its enclosing loop(s), e.g.: + * + * 1: for (...): // enclosing loop, header at 1 + * 2: for (...): // inner loop, header at 2 + * ... + * 3: if ...: + * goto 2b; // enters loop 2 without going through header 1 + * + * Such a bypass makes the enclosing loop irreducible, so its header is missing + * from the stack and both headers (1) and (2) need to be pushed onto stack. + * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds; + * the bypassed ancestors are irreducible and pushed as non-terminating. + * + * Assumes loop_stack_pop() has already truncated the stack to the common ancestor + * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack. + */ +static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *loop_stack = frame->loop_stack; + u32 missing_headers[LOOP_STACK_SIZE]; + u32 cnt = frame->loop_stack_cnt; + u32 num_missing = 0; + int h; + + *pushed = false; + + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) { + if (cnt && loop_stack[cnt - 1].loop_id == h) + break; + if (num_missing == LOOP_STACK_SIZE) + goto e2big; + missing_headers[num_missing] = h; + } + + if (num_missing == 0) + return 0; + if (cnt + num_missing > LOOP_STACK_SIZE) + goto e2big; + frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing, + sizeof(*frame->loop_stack)); + if (!frame->loop_stack) + return -ENOMEM; + loop_stack = frame->loop_stack; + + for (; num_missing; num_missing--, cnt++) { + h = missing_headers[num_missing - 1]; + loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h }; + if (env->log.level & BPF_LOG_LEVEL2) + verbose(env, "entering loop %d\n", h); + *pushed = true; + } + frame->loop_stack_cnt = cnt; + return 0; + +e2big: + verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx); + return -E2BIG; +} + +/* + * An exit from a loop can cross several nested loops, e.g.: + * + * 1: for (...): + * 2: for (...): + * 3: for (...): + * if ...: // before goto the loop stack is [1, 2, 3 ] + * goto 1b; // after goto it should become [1] + */ +static int loop_stack_pop(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + int new_cnt = 0; + int h, i; + + /* + * Walk the loop nest of insn_idx outwards (innermost first) and stop at + * the first loop that is present on the stack - that loop is the common + * ancestor and becomes the new top. If no enclosing loop is on the + * stack, drop all loops. + */ + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) { + for (i = frame->loop_stack_cnt - 1; i >= 0; i--) { + if (frame->loop_stack[i].loop_id == h) { + new_cnt = i + 1; + goto pop; + } + } + } + +pop: + if (env->log.level & BPF_LOG_LEVEL2) + for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--) + verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id); + frame->loop_stack_cnt = new_cnt; + return 0; +} + +static int maybe_clamp_scev_regs(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + + if (frame->loop_stack_cnt == 0) { + verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx); + return -EFAULT; + } + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (!entry->terminates) + return 0; + + return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state, + entry->max_header_count); +} + +static int handle_loop_entry_exit(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + struct bpf_loop_iters iters; + bool pushed, terminates; + int err; + + err = loop_stack_pop(env); + if (err) + return err; + + err = loop_stack_push(env, &pushed); + if (err) + return err; + + if (frame->loop_stack_cnt == 0) + return 0; + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (env->insn_idx != entry->loop_id) + return 0; + + /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */ + if (!pushed) + return maybe_clamp_scev_regs(env); + + err = bpf_compute_loop_iters(env, env->cur_state, &iters); + if (err < 0) + return err; + + terminates = err == 1; + if (!terminates) + return 0; + + /* Create a loop_entry checkpoint before widening */ + err = bpf_split_cur_state(env); + if (err) + return err; + + entry->min_header_count = iters.min_header_count; + entry->max_header_count = iters.max_header_count; + entry->entry_state = cur->parent; + entry->terminates = true; + err = bpf_widen_scev_regs(env, cur, cur->parent, &iters); + if (err < 0) + return err; + return 0; +} + static int do_check(struct bpf_verifier_env *env) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); @@ -19930,6 +20107,12 @@ static int do_check(struct bpf_verifier_env *env) struct bpf_insn_aux_data *insn_aux; int err; + if (signal_pending(current)) + return -EAGAIN; + + if (need_resched()) + cond_resched(); + /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; @@ -19976,6 +20159,15 @@ static int do_check(struct bpf_verifier_env *env) } } + /* + * Possibly widen the registers before creating a checkpoint + * in bpf_is_state_visited(). The next loop iteration will + * have a chance to hit this checkpoint and converge. + */ + err = handle_loop_entry_exit(env); + if (err) + return err; + if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); if (err < 0) @@ -20001,12 +20193,6 @@ static int do_check(struct bpf_verifier_env *env) return err; } - if (signal_pending(current)) - return -EAGAIN; - - if (need_resched()) - cond_resched(); - if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, @@ -20016,6 +20202,13 @@ static int do_check(struct bpf_verifier_env *env) do_print_state = false; } + if (bpf_loop_at_index(env, env->insn_idx) >= 0 && + cur_func(env)->loop_stack_cnt == 0) { + verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n", + env->insn_idx, bpf_loop_at_index(env, env->insn_idx)); + return -EFAULT; + } + if (env->log.level & BPF_LOG_LEVEL) { if (verifier_state_scratched(env)) print_insn_state(env, state, state->curframe); diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index dc7baa0f2458..84302a122ccf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -511,7 +511,7 @@ __naked void too_many_gotox_edges(void) SEC("socket") __description("gotox-edges-at-limit") -__success __retval(0) +__failure __msg("Too many nested loops") __naked void gotox_edges_at_limit(void) { asm volatile ( @@ -519,6 +519,10 @@ __naked void gotox_edges_at_limit(void) * 1000 gotox * 1000 targets = 1,000,000 CFG edges. At run * time, each block loads the next table entry and jumps to * the following block, so the program terminates. + * + * Every block can jump to every later block, so the CFG is + * a nest of ~1000 irreducible loops, deeper than the + * verifier's loop stack; the program is rejected. */ GOTOX_TABLE_BEGIN(1000) ".rept 1000;" From 088a5477a9bdb410169ff158a581bc2585db546c Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:53 -0700 Subject: [PATCH 423/660] bpf: Avoid widening registers that hinder exact stack-slot tracking For loops like this: int arr[10]; for (int i = 0; i < 10; i++) arr[i] = i; use(arr[i]); SCEV-based loop widening converts `i` from being enumerated 10 times as a constant value 0, 1, ..., 9 to being enumerated once as a range [0..9]. From the point of view of the main verification pass, this replaces constant-offset stack access with varying-offset stack access. The verifier tracks stack values accessed through varying offsets much less precisely than those accessed via constant offsets. For example, the read at use(arr[i]) would yield STACK_MISC. Precision might be lost for the following operations: - spills (BPF_ST/STX to the stack): stored value can no longer be recovered when read back, e.g. after the loop; - fills (BPF_LDX from the stack): a varying-offset read of a spilled pointer yields a SCALAR_VALUE, so the pointer is lost. - calls that construct objects on the stack (dynptr, iter, irq_flag, res_spin_lock): the stack argument is required to have a constant offset. This commit adds logic to avoid widening registers used by such instructions: - During the main SCEV computation stage, collect_store_base_regs(): - inspects instructions accessing the stack (spill, fill, call to a function listed in bpf_needs_fixed_stack_off), and - if there is a SCEV expression for a stack base address register, records which registers the expression depends on in the bpf_loop->store_base_reg field. - `store_base_reg`s accumulated for nested loops are propagated to outer loops as well. - During the loop-widening stage, bpf_compute_loop_iters() refuses to widen a loop when some of the registers the loop changes appear in store_base_regs. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-35-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 10 +++++ kernel/bpf/scev.c | 86 +++++++++++++++++++++++++++++++++++- kernel/bpf/verifier.c | 49 ++++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d760d2283ac5..308e779c3f74 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -708,6 +708,9 @@ struct bpf_loop_exit { int to; /* instruction outside the loop */ }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + struct bpf_loop { struct bpf_backedge backedges[MAX_BACKEDGES]; /* edges exiting from this loop, includes edges from nested loops */ @@ -717,6 +720,12 @@ struct bpf_loop { bool irreducible; bool backedges_overflow; bool exits_overflow; + /* + * Loop-entry registers used to compute stack addresses for loads, + * stores and calls requiring fixed stack offsets, in this and nested + * loops. + */ + unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; }; struct bpf_insn_aux_data { @@ -1876,6 +1885,7 @@ int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 05dd8e439609..adad32d8cb12 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -10,7 +10,7 @@ #include #include -#define REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) +#define REGS_NUM BPF_SCEV_REGS_NUM #define UNKNOWN_EXPR_ID 0 #define OPAQUE_EXPR_ID 1 @@ -1100,6 +1100,70 @@ static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct } } +/* + * Collect loop-entry registers referenced by expr 'id', best effort. + * DEPTH_LIMIT may leave dependencies unrecorded. This mask only avoids + * widening that would lose stack-access precision; missing dependencies + * may cause false rejections, but this is not a soundness issue. + */ +static void or_expr_regs(struct bpf_verifier_env *env, u32 id, unsigned long *mask) +{ + struct scev *scev = env->scev; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->exprs[id].op == REG) + __set_bit(scev->exprs[id].params[0], mask); + } +} + +/* Mask of argument registers (R1..R5) a call at 'idx' passes by register. */ +static u16 call_params_mask(struct bpf_verifier_env *env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct bpf_call_summary cs; + int n = bpf_get_call_summary(env, insn, &cs) ? cs.arg_slot_cnt : MAX_BPF_FUNC_REG_ARGS; + + return n ? GENMASK(BPF_REG_1 + n - 1, BPF_REG_1) : 0; +} + +/* + * For instructions like: + * - *(u64 *)(rBase + off) = rX + * - rX = *(u64 *)(rBase + off) + * - calls that construct objects on stack (e.g. dynptr_from_mem(rBase, ...)) + * When 'rBase' can be a stack pointer and is derived from some registers Rs + * defined at loop entry, record Rs into 'mask'. + */ +static void collect_store_base_regs(struct bpf_verifier_env *env, + struct env *cur_env, int idx, unsigned long *mask) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + u8 class = BPF_CLASS(insn->code); + u8 size = BPF_SIZE(insn->code); + u16 base_regs = 0; + u32 r; + + if (size == BPF_W || size == BPF_DW) { + if ((class == BPF_STX || class == BPF_ST) && insn->dst_reg != BPF_REG_FP) + base_regs |= BIT(insn->dst_reg); + else if (class == BPF_LDX && insn->src_reg != BPF_REG_FP) + base_regs |= BIT(insn->src_reg); + } + + if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL && + bpf_needs_fixed_stack_off(env, idx)) + base_regs |= call_params_mask(env, idx); + + base_regs &= aux[idx].stack_ptrs; + for (r = 0; r < MAX_BPF_REG; r++) + if (base_regs & BIT(r)) + or_expr_regs(env, cur_env->reg2expr[r], mask); +} + /* Find the topmost loop header containing idx inside cur_header, or -1 if none. */ static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header) { @@ -1184,6 +1248,7 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) struct bpf_iarray *succ; bool log_level2 = env->log.level & BPF_LOG_LEVEL2; int s, i, err, idx, succ_idx; + u32 r; if (log_level2) bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); @@ -1197,6 +1262,7 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) if (!header_env) return -ENOMEM; /* The freshly allocated environment has all expressions unknown. */ + bitmap_fill(cur_loop->store_base_regs, REGS_NUM); header_env->empty = false; return 0; } @@ -1233,6 +1299,9 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) */ if (log_level2) memcpy(old_env, cur_env, sizeof(*old_env)); + /* Pull the nested loop's stack-store base dependencies up. */ + for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], cur_loop->store_base_regs); forget_non_invariants(env, cur_env, idx); if (log_level2) log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); @@ -1254,6 +1323,7 @@ static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) for (;;) { if (log_level2) memcpy(old_env, cur_env, sizeof(*old_env)); + collect_store_base_regs(env, cur_env, idx, cur_loop->store_base_regs); err = transfer(env, cur_env, idx); if (err) goto out; @@ -2451,6 +2521,7 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta for (r = 0; r < REGS_NUM; r++) { struct bpf_reg_state *reg; u32 ra, r_scev, r_expr; + bool spill_base = false; if (!scev_reg_alive(env, st, r)) continue; @@ -2472,6 +2543,11 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta /* If SCEV for r is (linear ) */ if (is_simple_linear(scev, r_scev, &base_reg, &slope_imm) && reg && is_widenable_reg_type(reg)) { + /* Spills at varying offsets lose precision */ + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } continue; } @@ -2481,15 +2557,21 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta /* (any 1 (any 2 (any 3 4))) */ if (is_widenable_any(env, cur_func, header_env, r, r_scev)) { + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } continue; } - +cant_widen: if (log->level & BPF_LOG_LEVEL2) { r_expr = header_env->reg2expr[r]; bpf_log(log, "loop header at %d, can't widen ", insn_idx); log_reg(env, r); bpf_log(log, ", expr is "); log_expr(env, r_expr); + if (spill_base) + bpf_log(log, ", requires exact stack-offset tracking"); bpf_log(log, "\n"); } return 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a597a8c18972..2f190abe3448 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14257,6 +14257,55 @@ static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 of return *kfunc.flags & KF_SPINLOCK_SAFE; } +/* + * True if insn calls a helper/kfunc that requires one of its arguments to + * be a stack pointer with a constant offset. + */ +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + const struct bpf_func_proto *fn; + struct bpf_kfunc_desc *desc; + u32 *flags, btf_id; + int i; + + if (bpf_helper_call(insn)) { + if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) + return false; + } else if (bpf_pseudo_kfunc_call(insn)) { + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) + return false; + fn = &desc->proto; + } else { + return false; + } + + /* Both initialized and uninitialized stack dynptrs need a fixed offset. */ + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) + if (arg_type_is_dynptr(fn->arg_type[i])) + return true; + + /* vmlinux kfuncs only */ + if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0) + return false; + btf_id = insn->imm; + + flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog); + if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY))) + return true; + + if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] || + btf_id == special_kfunc_list[KF_bpf_local_irq_save] || + btf_id == special_kfunc_list[KF_bpf_local_irq_restore]) + return true; + + return false; +} + static bool is_sync_callback_calling_kfunc(u32 btf_id) { return is_bpf_rbtree_add_kfunc(btf_id); From a2052333e6f97b6326b3adb32f1eafc5d088bd4a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:54 -0700 Subject: [PATCH 424/660] bpf: Bpf_func_state size optimization Pack bpf_reg_state to bring it's size from 88 bytes to 80. Consequently, bpf_func_state changes from 1048 to 960. Thus bpf_func_state moves from 2K to 1K allocator bucket. 1K bucket is the one it fit before the SCEV series. With this optimization SCEV increases memory consumption on a subset of Meta internal programs by 3%, without this optimization the increase is 48%. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-36-bcba9bc53ac9@gmail.com --- include/linux/bpf_verifier.h | 4 ++-- kernel/bpf/states.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 308e779c3f74..54cb6d3d0653 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -184,9 +184,9 @@ struct bpf_reg_state { * other registers. Kept outside the metadata union for ID remapping * during state comparisons. */ - u32 map_uid; + u32 map_uid:31; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; + u32 precise:1; }; static inline s64 reg_smin(const struct bpf_reg_state *reg) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index c62ea992967b..953700822fa7 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1171,7 +1171,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, precise))) + offsetofend(struct bpf_reg_state, step))) return false; return true; } From 9fb8927b956b5e1e540756d3d786b522e5ca332a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:55 -0700 Subject: [PATCH 425/660] selftests/bpf: __msg_next tag for matching messages on consecutive lines Same as __msg, but expects the match on the line immediately after the last match, not just any later line. E.g.: __msg_next("a") __msg_next("c") This would match consecutive output "a\n" "c\n", but would not match output "a\n" "b\n" "c\n". Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-37-bcba9bc53ac9@gmail.com --- tools/testing/selftests/bpf/progs/bpf_misc.h | 6 ++++++ tools/testing/selftests/bpf/test_loader.c | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index f3dbc3b59bff..d7f706c3bc4a 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -43,6 +43,10 @@ * __msg_unpriv Same as __msg but for unprivileged mode. * __not_msg_unpriv Same as __not_msg but for unprivileged mode. * + * __msg_next Same as __msg but expects the match to be on the next + * __msg_next_unpriv log line compared to the last match, not just some line + * after the last match. + * * __stderr Message expected to be found in bpf stderr stream. The * same regex rules apply like __msg. * __stderr_unpriv Same as __stderr but for unpriveleged mode. @@ -145,6 +149,7 @@ #define __msg(msg) __test_tag("test_expect_msg=" msg) #define __not_msg(msg) __test_tag("test_expect_not_msg=" msg) +#define __msg_next(msg) __test_tag("test_expect_msg_next=" msg) #define __xlated(msg) __test_tag("test_expect_xlated=" msg) #define __jited(msg) __test_tag("test_jited=" msg) #define __failure __test_tag("test_expect_failure") @@ -153,6 +158,7 @@ #define __skip(reason) __test_tag("test_skip=" reason) #define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) #define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) +#define __msg_next_unpriv(msg) __test_tag("test_expect_msg_next_unpriv=" msg) #define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) #define __jited_unpriv(msg) __test_tag("test_jited_unpriv=" msg) #define __failure_unpriv __test_tag("test_expect_failure_unpriv") diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index a89890cd56d8..18d47e925400 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -508,6 +508,16 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next="))) { + err = __push_msg(msg, true, false, &spec->priv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= PRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next_unpriv="))) { + err = __push_msg(msg, true, false, &spec->unpriv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= UNPRIV; } else if ((msg = str_has_pfx(s, "test_jited="))) { if (arch_mask == 0) { PRINT_FAIL("__jited used before __arch_*"); @@ -998,6 +1008,7 @@ void validate_msgs(const char *log_buf, struct expected_msgs *msgs, no_match = !msg->negative && !match->start; wrong_line = !msg->negative && + match->start && msg->on_next_line && prev_match && prev_match->line + 1 != match->line; unexpected = msg->negative && match->start; From 0c7530110d274a6f3c66f6688af9fb5e87675b2a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:56 -0700 Subject: [PATCH 426/660] selftests/bpf: Bound UNIX socket path loops by sun_path size The bpf_iter/unix and skc_to_unix_sock tests index sun_path but bound that index by sizeof(struct sockaddr_un). This allows indices 108 and 109 into the 108-byte array. With SCEV loop widening, the variable-offset BTF access check rejects the resulting range. Bound the index by sizeof(sun_path) in both tests. Runtime behavior for valid UNIX addresses is unchanged. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-38-bcba9bc53ac9@gmail.com --- tools/testing/selftests/bpf/progs/bpf_iter_unix.c | 2 +- tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index df3d971b21e8..3061b148f338 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c @@ -70,7 +70,7 @@ int dump_unix(struct bpf_iter__unix *ctx) for (i = 1; i < len; i++) { /* unix_validate_addr() tests this upper bound. */ - if (i >= sizeof(struct sockaddr_un)) + if (i >= sizeof(unix_sk->addr->name->sun_path)) break; BPF_SEQ_PRINTF(seq, "%c", diff --git a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c index 4cfa42aa9436..c5c643064eb5 100644 --- a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c +++ b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c @@ -29,7 +29,7 @@ int BPF_PROG(unix_listen, struct socket *sock, int backlog) len = unix_sk->addr->len - sizeof(short); path[0] = '@'; for (i = 1; i < len; i++) { - if (i >= (int)sizeof(struct sockaddr_un)) + if (i >= (int)sizeof(unix_sk->addr->name->sun_path)) break; path[i] = unix_sk->addr->name->sun_path[i]; From 73e7927bdafada9d5306e43ef65563b28442f582 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:57 -0700 Subject: [PATCH 427/660] selftests/bpf: Test for stack-pointer subrange pruning Validate changes from: "bpf: allow subrange relations for PTR_TO_STACK in regsafe()". Add a test case that checks that a widened stack pointer with range [-72, -16] is considered a superset of a stack pointer with range [-64, -16]. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-39-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_stack_ptr.c | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 3e0bea9819ca..3e9d8fc606ae 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -586,4 +586,40 @@ __naked void stack_check_size_512_with_may_goto(void) } #endif +/* + * Verify that old PTR_TO_STACK state is considered a super-set of + * new PTR_TO_STACK state when new variable range is a sub-range + * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +/* + * r7 is widened to [-72, -16] at the loop header (insn 3), + * the loop body sees [-64, -8] after 'r7 += 8' + */ +__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("R7=fp(smin=smin32=-64,smax=smax32=-8") +/* + * back-edge state is clamped to the remaining iterations and pruned + * at the header, because [-64, -16] is within the widened [-72, -16] + */ +__msg("loop header at 3, clamping r7 to -64..-16 step 8") +__msg("from 5 to 3: safe") +__msg("processed 9 insns") +__naked void stack_ptr_subrange_in_loop(void) +{ + asm volatile (" \ + r7 = r10; \ + r7 += -72; \ + r6 = 0; \ + r7 += 8; \ + r6 += 1; \ + if r6 < 8 goto -3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; From c4db12b80c093118813256956cd8c9ff9cd83538 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:58 -0700 Subject: [PATCH 428/660] selftests/bpf: Tests for may_write stack-liveness tracking Branches to cover: - per-insn writes (record_stack_access_off): * off_cnt==1, fully covered slots -> def + may_def (equal) * partial coverage -> may_def only (no def) * off_cnt 2-4, precise offsets -> may_def only, per offset - whole-frame writes: * off_cnt==0 (offset lost) -> may_def SPIS_ALL * ARG_IMPRECISE (frame lost) -> may_def SPIS_ALL per frame - merge_instances(): * both passes touch a frame -> may_def union * one pass leaves a frame untouched -> may_def preserved - merge_may_write(): a callee's writes into ancestor frames are summarized as may_def at the caller's call instruction - analyze_subprog() -> may_def stack slot range when callback is unknown. Tests exercising them: - may_write_two_precise_offsets - off_cnt 2-4 precise multi-offset - imprecise_frame_write - cross-frame ARG_IMPRECISE write, plus its caller-side summarization - merge_preserves_may_write - map-value vs stack pointer at two callsites: union is preserved when one pass does not touch the frame - must_write_not_same_slot - off_cnt==0 whole-frame may_def - two_byte_write_no_kill - partial write, may_def without def - kfunc_iter_stack_liveness - full-slot def == may_def - helper_output_unknown_size - output size unknown to liveness, may_def without use or def - caller_stack_write, conditional_stx_in_subprog, shared_instance_must_write_overwrite - callee writes summarized as may_def at the call site - bpf_loop_two_callbacks - call to unknown callback marks full stack range as may_def. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-40-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 174 +++++++++++++++++- 1 file changed, 170 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 6e8f50618580..0b89c5776824 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -83,6 +83,31 @@ __naked void must_write_not_same_slot(void) : __clobber_all); } +SEC("socket") +__log_level(2) +__msg("stack use/def subprog#0 may_write_two_precise_offsets (d0,cs0):") +/* + * r1 is fp-8 or fp-16, with both offsets tracked precisely (off_cnt == 2). + * A write through it cannot set def, but marks both candidate slots as may_def. + */ +__msg("6: (7b) *(u64 *)(r1 +0) = r0 ; may_def: fp0-8 fp0-16") +__naked void may_write_two_precise_offsets(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = r10;" + "if r0 > 42 goto 1f;" + "r1 += -8;" + "goto 2f;" +"1:" + "r1 += -16;" +"2:" + "*(u64 *)(r1 + 0) = r0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __log_level(2) __msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") @@ -114,7 +139,7 @@ __log_level(2) __msg("stack use/def subprog#0 caller_stack_write (d0,cs0):") __msg("2: (85) call pc+1 ; may_def: fp0-8") __msg("stack use/def subprog#1 write_first_param (d1,cs2):") -__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8") +__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8 may_def: fp0-8") __naked void caller_stack_write(void) { asm volatile ( @@ -134,6 +159,49 @@ static __used __naked void write_first_param(void) ::: __clobber_all); } +/* + * Cross-frame imprecise write: imprecise_frame_writer() receives a pointer + * into the caller's frame (frame 0) but conditionally replaces it with a + * pointer into its own frame (frame 1). At the store the two are joined into + * ARG_IMPRECISE (frame is unknown, mask = {0,1}), so the offset is dropped and + * the whole of both candidate frames is conservatively marked as may_def. + */ +SEC("socket") +__log_level(2) +/* The callee's write into the caller frame is summarized at the call site. */ +__msg("stack use/def subprog#0 imprecise_frame_write (d0,cs0):") +__msg("2: (85) call pc+2 ; may_def: fp0-8..-{{(512|2048)}}") +__msg("stack use/def subprog#1 imprecise_frame_writer (d1,cs2):") +__msg("12: (7b) *(u64 *)(r1 +0) = r2 ; may_def: fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}") +__naked void imprecise_frame_write(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call imprecise_frame_writer;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void imprecise_frame_writer(void) +{ + asm volatile ( + "r6 = r1;" /* save arg: pointer into frame 0 */ + "call %[bpf_get_prandom_u32];" + "r1 = r6;" /* r1 = frame0-8 (arg) */ + "if r0 == 0 goto 1f;" + "r1 = r10;" + "r1 += -8;" /* r1 = frame1-8 (own frame) */ +"1:" + "r2 = 0;" + "*(u64 *)(r1 + 0) = r2;" /* join frame0-8 | frame1-8 -> imprecise */ + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __log_level(2) __msg("stack use/def subprog#0 caller_stack_read (d0,cs0):") @@ -1011,7 +1079,7 @@ __naked void four_byte_read_upper_half(void) */ SEC("socket") __log_level(2) -__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8 may_def: fp0-8") /* 2-byte write only partially covers the upper half: may_def, but no def. */ __msg("1: (6a) *(u16 *)(r10 -4) = 0 ; may_def: fp0-4h") __msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") @@ -1364,7 +1432,7 @@ __log_level(2) * fp-8 live at call: callee conditionally writes it, so the slot is not killed * (no def), but the conditional write surfaces as may_def at the call site. */ -__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") +__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8 may_def: fp0-8") __msg("4: (85) call pc+2 ; may_def: fp0-8") __msg("5: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") __naked void conditional_stx_in_subprog(void) @@ -2275,7 +2343,11 @@ static __used __naked void merge_leaf_read(void) SEC("socket") __log_level(2) __success -__msg("call bpf_loop#181 ; use: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}") +__msg("stack use/def subprog#0 bpf_loop_two_callbacks (d0,cs0):") +__msg("5: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("8: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("call bpf_loop#181 ; use: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}" + " may_def: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}{{$}}") __naked void bpf_loop_two_callbacks(void) { asm volatile ( @@ -2393,6 +2465,20 @@ __flag(BPF_F_TEST_STATE_FREQ) __msg("subprog#2 write_first_read_second:") __msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") __msg("18: (79) r0 = *(u64 *)(r2 +0) // r1=fp0-8 r2=fp0-16{{$}}") +/* + * The callee's write into the caller frame is summarized as may_def at the + * call sites, transitively across both frames (depth 2 -> 1 -> 0). The two + * call sites differ because the shared write_first_read_second instance + * accumulates the cross-pass union ({fp-8, fp-16}) at different points + * relative to each forwarding_rw's summarization. + */ +__msg("stack use/def subprog#0 shared_instance_must_write_overwrite (d0,cs0):") +__msg("7: (85) call pc+7 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("12: (85) call pc+2 ; use: fp0-8 may_def: fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs7):") +__msg("15: (85) call pc+1 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs12):") +__msg("15: (85) call pc+1 ; use: fp0-8 may_def: fp0-16") __msg("stack use/def subprog#2 write_first_read_second (d2,cs15):") /* * Shared across two callsites with swapped args (r1 is fp-8 on one pass, @@ -2441,6 +2527,65 @@ static __used __naked void write_first_read_second(void) ::: __clobber_all); } +/* + * merge_instances() must preserve may_write when one pass doesn't touch a frame. + * mvs_leaf is reached through a single callsite (mvs_mid) from two outer chains, + * so it is one shared (depth,callsite) instance analyzed twice and merged: + * - chain A passes r2 = map value, mvs_leaf's doesn't touch stack; + * - chain B passes r2 = caller stack slot fp-8. + * The merge must keep the stack pass's may_write even though the map pass left + * frame 0 untouched. + */ +SEC("socket") +__success +__log_level(2) +__msg("stack use/def subprog#2 mvs_leaf (d2,cs19):") +__msg("21: (7a) *(u64 *)(r2 +0) = 42 ; may_def: fp0-8") +__naked void merge_preserves_may_write(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 24) = r1;" /* map key */ + "r1 = %[map] ll;" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + /* chain A: r2 = map value (ARG_NONE, not stack) */ + "r1 = r10;" + "r1 += -16;" /* unused fp anchor */ + "r2 = r0;" + "call mvs_mid;" + /* chain B: r2 = caller stack slot fp-8 */ + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call mvs_mid;" +"1:" + "r0 = 0;" + "exit;" + : : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +static __used __naked void mvs_mid(void) +{ + asm volatile ( + "call mvs_leaf;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mvs_leaf(void) +{ + asm volatile ( + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + /* * Shared must_write when (callsite, depth) instance is reused. * Main calls fwd_to_stale_wr at two sites. fwd_to_stale_wr calls @@ -2949,3 +3094,24 @@ __naked void spill_below_512_stays_precise(void) "exit;" ::: __clobber_all); } + +/* Unknown-size outputs still carry a possible write, without a read or kill. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_probe_read_kernel{{.*}}; may_def: fp0-8{{$}}") +__naked void helper_output_unknown_size(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 7;" + "r0 += 1;" + "r2 = r0;" + "r1 = r10;" + "r1 += -8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel) + : __clobber_all); +} From 2c719872958c7892b1a86dbfb9d77f82b7e466f3 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:47:59 -0700 Subject: [PATCH 429/660] selftests/bpf: Tests for may_def marks of atomic RMW operations Liveness analysis should produce may_def marks for read-modify-write atomic operations: - lock *(u64 *)(r10 -8) += r1 - r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) - r1 = atomic64_xchg((u64 *)(r10 -8), r1) - r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r1) Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-41-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 0b89c5776824..43f616ea90a9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -2670,6 +2670,32 @@ __naked void load_acquire_dont_clear_dst(void) #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ +SEC("socket") +__log_level(2) +__success +__msg("lock *(u64 *)(r10 -8) += r1{{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__naked void atomic_rmw(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + ".8byte %[atomic_add];" + ".8byte %[atomic_fetch_add];" + ".8byte %[atomic_xchg];" + "r0 = 0;" + ".8byte %[atomic_cmpxchg];" + "exit;" + : + : __imm_insn(atomic_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + SEC("socket") __success __naked void imprecise_fill_loses_cross_frame(void) From 75a46f59e0985dd53fe7d5b2c4f42fd5e44ff144 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:00 -0700 Subject: [PATCH 430/660] selftests/bpf: Tests for stack pointer spills/fills through atomics Check stack write tracking for atomic operations: - Restored by 64-bit load-acquire, xchg, fetch_add and cmpxchg. - Spilled by store-release and restored by a plain load. - Restored from a caller's stack slot by xchg in a callee. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-42-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 43f616ea90a9..ae4c5062cfa6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -2696,6 +2696,127 @@ __naked void atomic_rmw(void) : __clobber_all); } +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +/* + * An 8-byte load-acquire from a spill slot restores the spilled register. + * The store through the restored pointer is a definite write to fp-8. + */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = load_acquire((u64 *)(r10 -16)){{.*}}use: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void load_acquire_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + ".8byte %[load_acquire_insn];" /* r2 = load_acquire(fp-16) */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -16)) + : __clobber_all); +} + +/* An 8-byte store-release spills the register, same as a plain store. */ +SEC("socket") +__log_level(2) +__success +__msg("store_release((u64 *)(r10 -16), r1){{.*}}def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void store_release_spills_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + ".8byte %[store_release_insn];" /* store_release(fp-16, r1): fp-16 = &fp-8 */ + "r2 = *(u64 *)(r10 - 16);" + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -16)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +/* Stack fills through xchg, fetch_add and cmpxchg instructions. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r10 -16), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r3 = atomic64_fetch_add((u64 *)(r10 -16), r3){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r3 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -16), r0, r4){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r0 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void atomic_rmw_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(fp-16, r2) */ + "*(u64 *)(r2 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r3 = 0;" + ".8byte %[atomic_fetch_add];" /* r3 = fetch_add(fp-16, r3) */ + "*(u64 *)(r3 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r0 = 0;" + "r4 = 0;" + ".8byte %[atomic_cmpxchg];" /* r0 = cmpxchg(fp-16, r0, r4) */ + "*(u64 *)(r0 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -16)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_3, -16)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_4, -16)) + : __clobber_all); +} + +/* xchg returning caller's &fp-8. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r1 +0), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}; may_def: fp0-8") +__naked void atomic_rmw_fills_parent_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r1 = r10;" + "r1 += -16;" + "call atomic_rmw_fills_parent_spilled_ptr_callee;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void atomic_rmw_fills_parent_spilled_ptr_callee(void) +{ + asm volatile ( + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(r1, r2) == caller's &fp-8 */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_2, 0)) + : __clobber_all); +} + SEC("socket") __success __naked void imprecise_fill_loses_cross_frame(void) From e60fb9b6b5beabe9f5cd7475768b04b5abe4d37f Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:01 -0700 Subject: [PATCH 431/660] selftests/bpf: Tests for map special cases in bpf_helper_stack_access_bytes() Tests for helper arguments whose access size or read/write effects depend on the map: - helper_map_output_known_size checks that for a known map access MEM_WRITE | MEM_UNINIT produces a must_write mark. - helper_map_output_merged_sizes checks that when maps with different value sizes merge, MEM_WRITE | MEM_UNINIT produces use and may_def marks over the maximum value size, without a def mark. - helper_bloom_peek_input checks that for bpf_map_peek_elem() on a bloom filter the buffer is treated as an input. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-43-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index ae4c5062cfa6..0fbc43a5d4f4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -3262,3 +3262,79 @@ __naked void helper_output_unknown_size(void) :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel) : __clobber_all); } + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64); +} queue_8b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64[2]); +} queue_16b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_BLOOM_FILTER); + __uint(max_entries, 16); + __type(value, __u64); +} bloom_8b SEC(".maps"); + +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; def: fp0-16") +__naked void helper_map_output_known_size(void) +{ + asm volatile ( + "r1 = %[queue_8b] ll;" + "r2 = r10;" + "r2 += -16;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* The maximum map value size is not a guaranteed output extent. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; use: fp0-16 fp0-24 may_def: fp0-16 fp0-24{{$}}") +__naked void helper_map_output_merged_sizes(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "call %[bpf_get_prandom_u32];" + "r1 = %[queue_8b] ll;" + "if r0 == 0 goto 1f;" + "r1 = %[queue_16b] ll;" +"1:" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm_addr(queue_16b), + __imm(bpf_get_prandom_u32), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* Bloom-filter peek treats buffer as an input despite the helper's output annotation. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_peek_elem{{.*}}; use: fp0-8{{$}}") +__naked void helper_bloom_peek_input(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = %[bloom_8b] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_peek_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(bloom_8b), __imm(bpf_map_peek_elem) + : __clobber_all); +} From 063c85f5d8c7c381d3d43fa42ada353058313ce4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:02 -0700 Subject: [PATCH 432/660] selftests/bpf: Tests for BTF-based global subprog stack liveness Check that passing a stack pointer to a no-op global callee produces: - use, def and may_def marks for a fixed-size memory argument. - No stack effects for an untrusted BTF pointer argument. This verifies that call effects follow the declared argument types rather than the callee's body. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-44-bcba9bc53ac9@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 0fbc43a5d4f4..3d88cb28b8fe 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -159,6 +159,52 @@ static __used __naked void write_first_param(void) ::: __clobber_all); } +struct cnt8 { long v; }; + +__noinline int global_noop_arg(struct cnt8 *p) +{ + return 0; +} + +SEC("socket") +__log_level(2) +/* Call to a global function: the argument is described by its declared type. */ +__msg("stack use/def subprog#0 global_callee_arg (d0,cs0):") +__msg("3: (85) call pc+{{.*}}; use: fp0-8 def: fp0-8 may_def: fp0-8") +__naked void global_callee_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_arg;" + "exit;" + ::: __clobber_all); +} + +struct task_struct { int pid; }; + +__noinline int global_noop_btf_arg(struct task_struct *p __arg_untrusted) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* Untrusted BTF pointer arguments need no caller stack state. */ +__msg("stack use/def subprog#0 global_callee_btf_arg (d0,cs0):") +__msg("3: (85) call pc+{{[0-9]+}}\n") +__naked void global_callee_btf_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_btf_arg;" + "exit;" + ::: __clobber_all); +} + /* * Cross-frame imprecise write: imprecise_frame_writer() receives a pointer * into the caller's frame (frame 0) but conditionally replaces it with a From a88078acc2e4921a53010739c3c218a537619b1f Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:03 -0700 Subject: [PATCH 433/660] selftests/bpf: Test for register liveness of subprogram arguments A static callee contributes the registers its code reads at the entry, a global callee its declared arguments. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-45-bcba9bc53ac9@gmail.com --- .../bpf/progs/compute_live_registers.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index b274a20ed7bd..b0aaf5961ac9 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -414,6 +414,35 @@ static __used __naked int aux1(void) ::: __clobber_all); } +static __noinline __used int static_two(int a, int b) +{ + return a; +} + +__noinline __used int global_two(int a, int b) +{ + return a; +} + +SEC("socket") +__log_level(2) +/* a static callee reads r1 only, a global callee its declared arguments */ +__msg("2: .1........ (85) call pc+") +__msg("5: .12....... (85) call pc+") +__naked void subprog_declared_args(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call static_two;" + "r1 = 1;" + "r2 = 2;" + "call global_two;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + SEC("socket") __log_level(2) __msg("0: .......... (b7) r1 = 1") From 2db5837f9adafbf7bd68aac4666400296b3bb8c7 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:04 -0700 Subject: [PATCH 434/660] selftests/bpf: Tests for register base/step arithmetic - step_arith{32,64}_no_overflow: non-overflowing +, *, << operations maintain step and base. - step_neg_value_range: Preserve the signed range of {-3, 0, 3, 6} when intersecting with a non-power-of-two step across zero. - step_arith{32,64}_overflow: Reset the stride and preserve valid bounds when a {32,64}-bit operation wraps. - step_mov32_truncate: Reset the stride when MOV32 discards nonzero upper bits from positive values. - step_mov32_negative: Reset the stride when MOV32 zero-extends the low 32 bits of negative signed values. - step_movsx: Reset the stride when sign extension changes the register value. - step_movsx_bounds_on_line: Keep bounds and stride when sign extension does not change the register value. - step_bounds_resync_after_arith: Snap bounds refined from var_off back onto the line. - step_linked_regs_add64: Translate both bounds and base when propagating a comparison's refinement through a scalar-ID delta. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-46-bcba9bc53ac9@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_bounds_step.c | 363 ++++++++++++++++++ 2 files changed, 365 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_bounds_step.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 460ad10ddc02..4f3af1cb998b 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -20,6 +20,7 @@ #include "verifier_basic_stack.skel.h" #include "verifier_bitfield_write.skel.h" #include "verifier_bounds.skel.h" +#include "verifier_bounds_step.skel.h" #include "verifier_bounds_deduction.skel.h" #include "verifier_bounds_deduction_non_const.skel.h" #include "verifier_bounds_mix_sign_unsign.skel.h" @@ -205,6 +206,7 @@ void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); } void test_verifier_bounds(void) { RUN(verifier_bounds); } +void test_verifier_bounds_step(void) { RUN(verifier_bounds_step); } void test_verifier_bounds_deduction(void) { RUN(verifier_bounds_deduction); } void test_verifier_bounds_deduction_non_const(void) { RUN(verifier_bounds_deduction_non_const); } void test_verifier_bounds_mix_sign_unsign(void) { RUN(verifier_bounds_mix_sign_unsign); } diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c new file mode 100644 index 000000000000..3a0d22aa4fb8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" + +/* Tests for the linear "base + step * k" description tracked per scalar register. */ + +#define __no_step __not_msg("step=") __msg("\n") + +/* + * Safe ALU64 arithmetic preserves a non-power-of-two stride and scales its base: + * step=0+3 -> step=1+3 -> step=2+6 -> step=4+12. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}step=0+3)") +__msg("r0 += 1 {{.*}}step=1+3)") +__msg("r0 *= 2 {{.*}}step=2+6)") +__msg("r0 <<= 1 {{.*}}step=4+12)") +__naked void step_arith64_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r0 *= 3; \ + r0 += 1; \ + r0 *= 2; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Safe ALU32 multiplication and addition preserve the stride and scale its base: + * step=0+3 -> step=1+3 -> step=2+6. MOV32 preserves the result. + * LSH32 currently resets the stride, but must preserve the correct upper bound. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 3 {{.*}}step=0+3)") +__msg("w0 += 1 {{.*}}step=1+3)") +__msg("w0 *= 2 {{.*}}step=2+6)") +__msg("w1 = w0 {{.*}}R0=scalar({{.*}}step=2+6) R1=scalar({{.*}}step=2+6)") +__msg("w0 <<= 1 {{.*}}smax=umax=smax32=umax32=3064,") __no_step +__naked void step_arith32_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + w0 *= 3; \ + w0 += 1; \ + w0 *= 2; \ + w1 = w0; \ + w0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-crossing range with a non-power-of-2 step. After "*= 3; += -3" the value + * set is {-3, 0, 3, 6}. The line description is tracked in signed space, so the + * intersection keeps smax=6. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -3 {{.*}}smax=smax32=6,step=0+3)") +__naked void step_neg_value_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r0 += -3; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * 0x49249249 * 7 wraps to U32_MAX; inferring step=0+7 would incorrectly + * cap the result at 0xfffffffc instead of U32_MAX. + * Adding 4 to {0xfffffffc, U32_MAX} wraps to {0, 3}; keeping step=1+3 + * incorrectly narrows the result to 1. + * Shifting the same input left by 1 gives {0xfffffff8, 0xfffffffe}; + * inferring step=0+6 incorrectly narrows the result to 0xfffffffc. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 7 {{.*}}umax=0xffffffff,") __no_step +__msg("r2 = r0 {{.*}}R0=scalar({{.*}}step=0+3) R2=scalar({{.*}}step=0+3)") +__msg("w0 += 4 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__msg("w2 <<= 1 {{.*}}smax=umax=umax32=0xfffffffe,") __no_step +__naked void step_arith32_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w0 *= 7; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0xfffffffc ll; \ + r0 += r1; \ + r2 = r0; \ + w0 += 4; \ + w2 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Multiplying {0x2aaaaaaaaaaaaaab, 0x2aaaaaaaaaaaaaac} by 3 crosses S64_MAX + * and gives {S64_MIN + 1, S64_MIN + 4}. Keeping step=0+3 excludes both. + * + * Adding 5 to {S64_MAX - 4, S64_MAX - 1} gives {S64_MIN, S64_MIN + 3}. + * Updating the line to step=2+3 incorrectly narrows it to S64_MIN + 1. + * + * Adding -5 to {S64_MIN + 1, S64_MIN + 4} wraps to + * {S64_MAX - 3, S64_MAX}. The stale base 0 modulo 3 excludes S64_MAX. + * + * Shifting {0x4000000000000008, 0x400000000000000b} left by 1 gives + * {S64_MIN + 16, S64_MIN + 22}. Keeping step=0+6 excludes both. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}smax=0x8000000000000004,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 += 5 {{.*}}smax=0x8000000000000003,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 += -5 {{.*}}umax=0x7fffffffffffffff,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 <<= 1 {{.*}}smax=0x8000000000000016,") __no_step +__naked void step_arith64_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r2 = r0; \ + r1 = 0x2aaaaaaaaaaaaaab ll; \ + r0 += r1; \ + r0 *= 3; \ + r2 *= 3; \ + r0 = r2; \ + r1 = 0x7ffffffffffffffb ll; \ + r0 += r1; \ + r0 += 5; \ + r0 = r2; \ + r1 = 0x8000000000000001 ll; \ + r0 += r1; \ + r0 += -5; \ + r0 = r2; \ + r1 = 0x4000000000000008 ll; \ + r0 += r1; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {2^32, 2^32 + 3} to {0, 3}. The old base 1 modulo 3 + * excludes both results. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += r1 {{.*}}step=1+3)") +__msg("w0 = w0 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__naked void step_mov32_truncate(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0x100000000 ll; \ + r0 += r1; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {-4, -1} to {0xfffffffc, U32_MAX}. The old base 2 + * modulo 3 excludes U32_MAX. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -4 {{.*}}step=2+3)") +__msg("w0 = w0 {{.*}}umax=0xffffffff,") __no_step +__naked void step_mov32_negative(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r0 += -4; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-extending {128, 131} from s8 gives {-128, -125}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -127. + * + * Sign-extending {32768, 32771} from s16 gives {-32768, -32765}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -32767. + * + * Sign-extending {2^31, 2^31 + 3} from s32 gives {S32_MIN, S32_MIN + 3}. + * Keeping base 2 modulo 3 incorrectly narrows the result to S32_MIN + 1. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 128 {{.*}}step=2+3)") +__msg("r0 = (s8)r0 {{.*}}smax=smax32=-125,") __no_step +__msg("r0 += 32768 {{.*}}step=2+3)") +__msg("r0 = (s16)r0 {{.*}}smax=smax32=-32765,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 = (s32)r0 {{.*}}smax=0xffffffff80000003,") __no_step +__naked void step_movsx(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r2 = r0; \ + r0 += 128; \ + r0 = (s8)r0; \ + r0 = r2; \ + r0 += 32768; \ + r0 = (s16)r0; \ + r0 = r2; \ + r1 = 0x80000000 ll; \ + r0 += r1; \ + r0 = (s32)r0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * R1 = R0 + 1 shares R0's scalar ID. Refining R0 to {0, 3, 6} must + * translate its line as well as its bounds when updating R1 to {1, 4, 7}. + * Copying R0's base 0 modulo 3 excludes the reachable value 7. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 += 1 {{.*}}step=1+3)") +__msg("if r0 > 0x6 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=7,{{.*}}step=1+3)") +__naked void step_linked_regs_add64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r1 = r0; \ + r1 += 1; \ + if r0 > 6 goto l_out_%=; \ + r0 = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r0 += 8 -> [8, 176] step=8+24 + * r0 & 0x10 -> [32, 176] step=8+24 (var_off alone: umin=16) + * r0 > 80 -> [104, 176] step=8+24 (var_off alone: umin=88) + * r1 += 24 -> [128, 200] step=8+24 + * Bounds and step stay in sync after each refinement. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 8 {{.*}}smin=umin=smin32=umin32=8,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("from 4 to 6: R0=scalar(smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 = r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=umin=smin32=umin32=104,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 += 24 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=128,smax=umax=smax32=umax32=200,{{.*}}step=8+24)") +__naked void step_bounds_resync_after_arith(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r0 += 8; \ + if r0 & 0x10 goto l_set_%=; \ + goto l_out_%=; \ +l_set_%=: \ + if r0 <= 80 goto l_out_%=; \ + r1 = r0; \ + r1 += 24; \ + *(u64 *)(r10 - 8) = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r1 = (s16)r0 -> [0, 168] step=0+24 + * r0 += -24 -> [-24, 144] step=0+24 + * r2 = (s16)r0 -> [-24, 144] step=0+24 + * r3 = (s8)r0 -> [-128, 127] no step + * w4 = (s16)w1 -> [0, 168] step=0+24 + * w5 = (s8)w1 -> [-128, 127] no step + * Bounds and step stay in sync, the step is dropped when the value does not fit. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 = (s16)r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)") +__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") __no_step +__msg("w4 = (s16)w1 {{.*}}R4=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,{{.*}})") __no_step +__naked void step_movsx_bounds_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r1 = (s16)r0; \ + r0 += -24; \ + r2 = (s16)r0; \ + r3 = (s8)r0; \ + w4 = (s16)w1; \ + w5 = (s8)w1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; From 8c5f6dfefdf17f42979e2ca0fe13f80dd80b34df Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:05 -0700 Subject: [PATCH 435/660] selftests/bpf: Tests for register base/step state pruning Tests for the base/step reasoning in states.c:range_within(): - step_prune_hit_multiple: prune a step-4 range contained in a cached step-2 range when the scalar is used to address the stack. - step_prune_miss_non_multiple: do not let a cached step-2 range hide a step-3 path reaching division by zero. - step_prune_hit_const_on_line: prune constant 6 against a cached base-0, step-3 range. - step_prune_miss_const_off_line: do not prune constant 7 against that range even though its bounds and tnum admit the value. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-47-bcba9bc53ac9@gmail.com --- .../bpf/progs/verifier_bounds_step.c | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c index 3a0d22aa4fb8..c9ee1c7efcbb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c @@ -360,4 +360,185 @@ __naked void step_movsx_bounds_on_line(void) : __clobber_all); } +struct step_val { + __u8 data[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct step_val); +} step_map SEC(".maps"); + +/* Old register [4..130, step 2] should prune cur register [8..68, step 4]. */ +SEC("socket") +__success __log_level(2) +__msg("7: (27) r6 *= 4 ; R6=scalar({{.*}}umin32=8,{{.*}}umax32=68,{{.*}},step=0+4)") +__msg("10: (27) r7 *= 2 ; R7=scalar({{.*}}umin32=4,{{.*}}umax32=130,{{.*}},step=0+2)") +__msg("11: (25) if r0 > 0x2a goto pc+1") +__msg("from 11 to 13: safe") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_hit_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 += 2; \ + r6 *= 4; \ + r7 &= 0x3f; \ + r7 += 2; \ + r7 *= 2; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=2 explored first, step=4 explored next */ \ +1: r0 = r10; \ + r6 = -r6; \ + r0 += r6; \ + *(u8 *)(r0 + 0) = 7; /* force r6 precise */ \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Old register [0..189, step 3] should not prune cur register [0..30, step 2]. + * Bounds and tnum containment allow pruning; step divisibility must prevent it. + */ +SEC("socket") +__success __log_level(2) +__msg("6: (27) r6 *= 2 ; R6=scalar({{.*}}smin32=0,{{.*}}umax32=30,{{.*}},step=0+2)") +__msg("8: (27) r7 *= 3 ; R7=scalar({{.*}}smin32=0,{{.*}}umax32=189,{{.*}},step=0+3)") +__msg("9: (25) if r0 > 0x2a goto pc+1") +__msg("17: (95) exit") +__msg("from 9 to 11: R6=scalar({{.*}},step=0+2)") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_miss_non_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 *= 2; \ + r7 &= 0x3f; \ + r7 *= 3; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=3 explored first, step=2 explored next */ \ +1: \ + r2 = r10; \ + r6 += 1; \ + r6 = -r6; \ + r2 += r6; \ + *(u8 *)(r2 + 0) = 7; /* force r6 precise */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Constant current register lying on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 6. range_within() takes the + * constant branch: imod(6, 3) == base 0, so cur is on the line and the + * (precise) state is pruned -> "safe". + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 6") /* current path: const 6, on the line */ +__msg("17: safe") /* pruned at join: imod(6, 3) == 0 */ +__naked void step_prune_hit_const_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 6; /* cur: const on line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + +/* + * Constant current register NOT on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 7. imod(7, 3) == 1 != base 0, + * so range_within() fails and the join is traversed again. A power-of-two + * step is avoided on purpose: with step 3 the tnum is loose enough to admit + * 7, so imod() is + * the sole check that rejects it. No failure shape is possible here: 7 is + * within the line's bounds and tnum, and the cached line path already + * verifies the whole outro, so an eager prune could not miss an error. + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 7") /* current path: const 7, off the line */ +/* not pruned: current continues past the join with the constant offset 7 */ +__msg("19: R0=map_value({{.*}}imm=7) R1=7") +__naked void step_prune_miss_const_off_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 7; /* cur: const off line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; From e187fac8ac6bc1a73647938e600da91c5edf2624 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:06 -0700 Subject: [PATCH 436/660] selftests/bpf: Tests for varying offset access to PTR_TO_BTF_ID Verifier tests for accessing arrays at a varying offset through a PTR_TO_BTF_ID pointer, e.g. a[i].b where 'i' is a register with a varying value: - bpf_obj_new() is used to obtain a PTR_TO_BTF_ID to a program-defined struct - bpf_get_current_task_btf() is used to obtain a PTR_TO_BTF_ID from the kernel BTF. The cases cover: - in-bounds access to a field of an array of structs; - out-of-bounds access where the maximal offset runs past the array; - a scalar (long) array with a matching stride; - a byte array (step 1); - a 4-byte read whose maximal offset spills past the array end; - an unaligned step that is not a whole number of elements; - both dimensions of a 2D array; - an array of pointers within a program-allocated object, where reading a pointer member yields a SCALAR_VALUE but the varying offset is still validated against the array bounds (in-bounds and out-of-bounds); - a varying access into an array embedded in a kernel BTF type (task_struct's comm[]), covering the non-allocated, trusted-pointer path (in-bounds and out-of-bounds); - an array nested below a struct member at a non-zero offset, exercising the offset telescoping done while walking across a struct boundary (in-bounds and out-of-bounds); - a trailing flexible array member, whose unbounded tail makes a varying access have no upper bound to exceed. A program-defined struct cannot be used here because bpf_obj_new() forbids flexible array members, so a kernel type (struct vring_used, with its 'ring[]' tail) is used (in-bounds, and an unaligned step that is not a whole number of elements). Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-48-bcba9bc53ac9@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_btf_array_access.c | 576 ++++++++++++++++++ 2 files changed, 578 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_btf_array_access.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 4f3af1cb998b..2f05a0ce829f 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -27,6 +27,7 @@ #include "verifier_bpf_get_stack.skel.h" #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" +#include "verifier_btf_array_access.skel.h" #include "verifier_btf_ctx_access.skel.h" #include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" @@ -213,6 +214,7 @@ void test_verifier_bounds_mix_sign_unsign(void) { RUN(verifier_bounds_mix_sign_u void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } +void test_verifier_btf_array_access(void) { RUN(verifier_btf_array_access); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c new file mode 100644 index 000000000000..be001cc77f81 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_experimental.h" +#include "bpf_misc.h" + +/* + * Variable-offset access into arrays within a BTF access chain, e.g. a[i].b, + * where 'i' is a register with a varying offset. The register's step has to + * match the stride of one of the arrays crossed while walking the access + * chain, and the maximal possible offset has to stay within that array. + * + * A program-allocated object (bpf_obj_new) is used as the source of a + * PTR_TO_BTF_ID with a shape we fully control. + */ +struct inner { + int a; + int b; +}; + +struct outer { + struct inner arr[8]; /* off 0, stride 8, size 64 */ + char bytes[32]; /* off 64, stride 1, size 32 */ + long longs[8]; /* off 96, stride 8, size 64 */ + int grid[4][4]; /* off 160, 16 ints, size 64 */ +}; + +/* arr[i].b, i in [0, 7], 4-byte read: stays within arr. */ +SEC("syscall") +__success +int arr_field_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* arr[i].b, i in [0, 15]: max offset runs past the end of arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_field_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* longs[i], i in [0, 7], 8-byte read: scalar array with stride 8. */ +SEC("syscall") +__success +int long_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 96); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * bytes[i], i in [0, 31], read as 4 bytes: min_off is valid within the array, + * but the 4-byte read at max_off runs past the array end. Exercises the + * size-aware bound in btf_struct_access(). + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int byte_array_size_spanning(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* bytes[i], i in [0, 31], 1-byte read: step 1 matches the char array stride. */ +SEC("syscall") +__success +int byte_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * arr[] accessed with step 4 (half of sizeof(struct inner)): the step is not + * a whole number of elements, so no crossed array has a matching stride. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[i][0], i in [0, 3]: steps along the outer dimension of a 2D array + * (stride 16). __btf_resolve_size() linearizes the array; the step is a + * multiple of the innermost element size (4). + */ +SEC("syscall") +__success +int grid_outer_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 16; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[0][i], i in [0, 3]: steps along the inner dimension of a 2D array + * (stride 4, the innermost element size). + */ +SEC("syscall") +__success +int grid_inner_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * An array of pointers within a program-allocated object. Reading a pointer + * member of a local object yields a SCALAR_VALUE, but the varying offset still + * has to be validated against the array bounds. + */ +struct with_ptrs { + struct inner *parr[8]; /* off 0, stride 8 (sizeof ptr), size 64 */ +}; + +/* parr[i], i in [0, 7], 8-byte read: stays within parr. */ +SEC("syscall") +__success +int ptr_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * parr[i], i in [0, 15]: max offset runs past the end of parr. Without the + * variable-offset check on the WALK_PTR path this would be wrongly accepted. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct with_ptrs") +int ptr_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * The cases above all use bpf_obj_new() (PTR_TO_BTF_ID | MEM_ALLOC). Exercise + * the trusted kernel-BTF path too, using a task_struct from + * bpf_get_current_task_btf() and its embedded char comm[] array. The comm + * offset is folded into the pointer as a constant before the varying index. + */ + +/* task->comm[i], i in [0, 15], 1-byte read: stays within comm. */ +SEC("syscall") +__success +int kernel_btf_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 15; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* task->comm[i], i in [0, 31]: max offset runs past comm. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct task_struct") +int kernel_btf_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* + * Array nested below a struct member at a non-zero offset. The walk descends + * through 'm' (resetting the running offset) before reaching 'arr', exercising + * the array_start = min_off - arrays[i].off telescoping across a WALK_STRUCT + * dive. + */ +struct mid { + struct inner arr[8]; /* stride 8, size 64 */ +}; + +struct nest { + long pad; /* off 0 */ + struct mid m; /* off 8 */ +}; + +/* m.arr[i].b, i in [0, 7], 4-byte read: stays within m.arr. */ +SEC("syscall") +__success +int nested_struct_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* m.arr[i].b, i in [0, 15]: max offset runs past the end of m.arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct nest") +int nested_struct_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * A trailing flexible array member makes the struct tail an unbounded region, + * so a varying access into it has no upper bound to exceed (mirrors how + * unix_address.name[] is accessed via sun_path[i]). + * + * A custom (program) BTF struct can only be reached as a PTR_TO_BTF_ID through + * bpf_obj_new(), which does not allow flexible array members. So use a kernel + * type via an untrusted PTR_TO_BTF_ID (bpf_core_cast()). struct vring_used ends + * with a flexible array 'ring[]' of struct vring_used_elem { __virtio32 id, len; }, + * mirroring a 'struct foo { int a; int b; }' flexible array. + */ + +/* ring[i].len, i in [0, 63], 4-byte read: the flexible array has no upper bound. */ +SEC("syscall") +__success +int flex_array_field(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 63; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 8); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * ring[] accessed with step 4 (half of sizeof(struct vring_used_elem)): the + * step is not a whole number of elements, so the flexible array stride does + * not match. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct vring_used") +int flex_array_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * Access either name[0].sun_path[0] or one byte past sun_path. + * Step 108 does not match sizeof(struct sockaddr_un), which is 110. + * It matches the inner char array's stride, but exceeds its bounds. + * The flexible-tail exemption must not apply to that fixed array. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct unix_address") +int flex_array_inner_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct unix_address *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct unix_address); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 1; \ + r2 *= %[len]; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + %[off]); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(len, sizeof_field(struct sockaddr_un, sun_path)), + __imm_const(off, offsetof(struct unix_address, name) + + offsetof(struct sockaddr_un, sun_path)) + : "r1", "r2"); + return val; +} + +typedef int array_element_t; +typedef array_element_t array_type_t[4]; + +struct typedef_array { + array_type_t values; +}; + +/* + * Resolving values' size crosses both the array and element typedefs. + * Record the array ID, not the int ID left by element-type resolution. + * values[i], i in [0, 3], must pass the variable-offset array check. + */ +SEC("syscall") +__success +int typedef_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct typedef_array *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= %[elem_size]; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(elem_size, sizeof(array_element_t)) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +char _license[] SEC("license") = "GPL"; From 951538da0fa9a5fcdceb892aaa95af9151f10a6a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:07 -0700 Subject: [PATCH 437/660] selftests/bpf: Tests for loop hierarchy computation Test cases covering the following branches in bpf_compute_loops(): - Case B: simple backedge creating a loop (loop_single) - Case B: two independent loops (loop_two_independent) - Case B + D: nested loops where the inner header's loop_header points to the outer header (loop_nested) - Case C: diamond CFG with no loops (fwd_edges_no_loop) - Case D: sibling inner loops within one outer loop (loop_nested_siblings) - Three levels of loop nesting (loop_three_levels) - Loop with an if-else body containing forward branches (loop_with_if_else) - Case E: An irreducible loop (loop_irreducible) - A self-loop (loop_self) - A test with sibling loops nested in outer loops (loop_nested_siblings_common_ancestors) - A test case with outer loop's backedge originating from an inner loop (loop_outer_backedge_from_inner). Cases B, C, D and E are described in loops.c:compute_loops_in_subprog(). Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-49-bcba9bc53ac9@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_loop_hierarchy.c | 346 ++++++++++++++++++ 2 files changed, 348 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 2f05a0ce829f..6b60d7720e2a 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -74,6 +74,7 @@ #include "verifier_live_stack.skel.h" #include "verifier_liveness_exp.skel.h" #include "verifier_load_acquire.skel.h" +#include "verifier_loop_hierarchy.skel.h" #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" @@ -261,6 +262,7 @@ void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } void test_verifier_live_stack(void) { RUN(verifier_live_stack); } void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } +void test_verifier_loop_hierarchy(void) { RUN(verifier_loop_hierarchy); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c new file mode 100644 index 000000000000..0608e983d7fc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" + +/* + * kernel/bpf/loops.c:compute_loops_in_subprog() distinguishes between + * the following cases: + * - B: backedge -> simple loop + * - C: cross edge to non-loop node -> no-op + * - D: edge to node whose header is in DFS path -> nested loop + * - E: edge to node whose header is NOT in DFS path -> irreducible + * + * Below test cases cover the above branches in various combinations. + */ + +/* Case B: single bounded loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (95) exit") +__naked void loop_single(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B: two independent loops at the same nesting level. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 4{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 2 0 1: {{.*}} (07) r0 += 1") +__msg(" 2 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (b7) r1 = 0") +__msg(" 1 3 4: {{.*}} (07) r1 += 1") +__msg(" 1 4 4 5: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_two_independent(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 10 goto 2b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B + D: nested loops. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") /* outer loop header */ +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") /* outer loop insn */ +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") /* inner loop header */ +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") /* inner loop insn */ +__msg(" 1 1 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") /* outer loop insn */ +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_nested(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case C: forward edges, no loops. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32") +__msg(" 0 1: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg(" 1 2: {{.*}} (b7) r0 = 2") +__msg(" 2 3: {{.*}} (05) goto pc+1") +__msg(" 1 4: {{.*}} (b7) r0 = 3") +__msg(" 1 5: {{.*}} (95) exit") +__naked void fwd_edges_no_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 2; \ + goto 2f; \ +1: r0 = 3; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Case B + D: two sibling inner loops within one outer loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 6, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg(" 1 1 4 5: {{.*}} (b7) r2 = 0") +__msg(" 1 1 5 6: {{.*}} (07) r2 += 1") +__msg(" 1 6 6 7: {{.*}} (a5) if r2 < 0x5 goto pc-2") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_nested_siblings(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 5 goto 3b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Three levels of nesting. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 5, nested in 3") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (b7) r2 = 0") +__msg(" 1 3 4 5: {{.*}} (07) r2 += 1") +__msg(" 1 5 5 6: {{.*}} (a5) if r2 < 0x3 goto pc-2") +__msg(" 1 3 6 7: {{.*}} (a5) if r1 < 0x5 goto pc-5") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_three_levels(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Loop with an if-else body (forward branch inside loop, Case D). */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (bf) r1 = r0") +__msg(" 1 1 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg(" 1 1 3 4: {{.*}} (b7) r1 = 1") +__msg(" 1 1 3 5: {{.*}} (0f) r0 += r1") +__msg(" 1 1 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg(" 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case E: irreducible loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 3, irreducible") +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg(" 0 1: {{.*}} (b7) r1 = 0") +__msg(" 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg(" 1 2 3: {{.*}} (b7) r1 = 1") +__msg(" 1 3 3 4: {{.*}} (05) goto pc+1") +__msg(" 2 5: {{.*}} (b7) r1 = 2") +__msg(" 1 3 2 6: {{.*}} (0f) r0 += r1") +__msg(" 1 3 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg(" 7 8: {{.*}} (95) exit") +__naked void loop_irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__failure +__log_level(2) +__msg("loop at 1") +__msg("infinite loop detected at insn 1") +__naked void loop_self(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loops at 5 and 7 are nested inside loops at 3 and 1. + * The edge 6 -> 7 exits loop at 5 and enters loop at 7. + * Check that it is logged only as an exit from loop at 5. + * + * 0: r0 = 0; + * do { + * 1: r0++; + * 2: r1 = 0; + * do { + * 3: r1++; + * 4: r2 = 0; + * do { + * 5: r2++; + * 6: } while (r2 < 3); + * do { + * 7: r2--; + * 8: } while (r2 != 0); + * 9: } while (r1 < 4); + * 10: } while (r0 < 5); + * 11: return r0; + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg_next(" backedge from 10, latch at 10") +__msg_next(" exit from 10 to 11") +__msg_next("loop at 3, nested in 1") +__msg_next(" backedge from 9, latch at 9") +__msg_next(" exit from 9 to 10") +__msg_next("loop at 5, nested in 3") +__msg_next(" backedge from 6, latch at 6") +__msg_next(" exit from 6 to 7") +__msg_next("loop at 7, nested in 3") +__msg_next(" backedge from 8, latch at 8") +__msg_next(" exit from 8 to 9") +__naked void loop_nested_siblings_common_ancestors(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ +4: r2 += -1; \ + if r2 != 0 goto 4b; \ + if r1 < 4 goto 2b; \ + if r0 < 5 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The outer loop backedge (4 -> 2) originates in an inner loop. + * For now, latch should not be identified in such cases. + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 7") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 6, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 7") +__naked void loop_outer_backedge_from_inner(void) +{ + asm volatile (" \ + r6 = 99; \ + r2 = 2; \ +2: r6 += 1; /* loop at 2 */ \ +3: r2 += 1; /* loop at 3 */ \ +4: if r2 == 3 goto 2b; \ +5: if r6 > 100 goto 7f; \ + goto 3b; \ +7: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; From aa8964a8e5f1fe3e5e182990bf8ae4ac122da749 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:08 -0700 Subject: [PATCH 438/660] selftests/bpf: Tests for immediate dominator computation Coverage: - straight-line code with an ldimm64 in the middle; - an asymmetric if-then-else diamond (unequal-depth idoms_intersect()); - a simple loop, a loop with an if-else body, and nested loops; - a loop header with two back edges (three predecessors); - an irreducible CFG (fixpoint convergence); - immediate dominators across subprogram boundaries, including a callee whose entry instruction is itself a loop header; - a self-loop; - an indirect jump (gotox) with a two-entry jump table. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-50-bcba9bc53ac9@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../selftests/bpf/progs/verifier_idoms.c | 377 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_idoms.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 6b60d7720e2a..b45916e04fef 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -58,6 +58,7 @@ #include "verifier_helper_packet_access.skel.h" #include "verifier_helper_restricted.skel.h" #include "verifier_helper_value_access.skel.h" +#include "verifier_idoms.skel.h" #include "verifier_int_ptr.skel.h" #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" @@ -246,6 +247,7 @@ void test_verifier_helper_access_var_len(void) { RUN(verifier_helper_access_var_ void test_verifier_helper_packet_access(void) { RUN(verifier_helper_packet_access); } void test_verifier_helper_restricted(void) { RUN(verifier_helper_restricted); } void test_verifier_helper_value_access(void) { RUN(verifier_helper_value_access); } +void test_verifier_idoms(void) { RUN(verifier_idoms); } void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } diff --git a/tools/testing/selftests/bpf/progs/verifier_idoms.c b/tools/testing/selftests/bpf/progs/verifier_idoms.c new file mode 100644 index 000000000000..fdee6bced77b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_idoms.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* + * The immediate dominator is printed by log_program() as the numeric column + * immediately before the ":" field of every "Program dump" line: + * + * Program dump (scc? loop_header? idom insn#: live_regs_before): + * -1 0: ....... (b7) r0 = 0 <- idom(0) = -1 (subprog entry) + * 0 1: ....... (07) r0 += 1 <- idom(1) = 0 + * ^^^^^^ + * idom insn# + */ + +/* Straight-line code, with an ldimm64 in the middle. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (18) r1 = 0x1122334455667788") +__msg("{{.*}} 1 3: {{.*}} (07) r0 += 1") +__msg("{{.*}} 3 4: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 4 5: {{.*}} (95) exit") +__naked void straight_line_ldimm64(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 0x1122334455667788 ll; \ + r0 += 1; \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Asymmetric if-then-else diamond: the "then" arm is 5 instructions long, the + * "else" arm is a single instruction. The merge point (insn 8) is dominated by + * the branch (insn 1), not by either arm. This forces idoms_intersect() to walk + * the two predecessors up unequal postorder depths before they meet. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (25) if r0 > 0x0 goto pc+5") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 3") +__msg("{{.*}} 4 5: {{.*}} (b7) r1 = 4") +__msg("{{.*}} 5 6: {{.*}} (05) goto pc+1") +__msg("{{.*}} 1 7: {{.*}} (b7) r1 = 9") +__msg("{{.*}} 1 8: {{.*}} (95) exit") +__naked void asymmetric_diamond(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r1 = 1; \ + r1 = 2; \ + r1 = 3; \ + r1 = 4; \ + goto 2f; \ +1: r1 = 9; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Simple loop: the header (insn 1) is dominated by the pre-header (insn 0). The + * back edge (insn 2 -> insn 1) must not change idom(1); the back-edge + * predecessor folds up via idoms_intersect(). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg("{{.*}} 2 3: {{.*}} (95) exit") +__naked void simple_loop(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop with an if-else in its body. The in-loop merge point (insn 5) is + * dominated by the in-loop branch (insn 3), combining a back-edge intersect + * with a forward-diamond intersect. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (bf) r1 = r0") +__msg("{{.*}} 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 5: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else_body(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Nested loops: the idom chain runs inner-header -> outer-body -> outer-header + * -> pre-header. Exercises intersect across back edges at two nesting depths. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 2 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void nested_loops(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop header (insn 1) with two back edges (from insn 4 and insn 5), i.e. three + * predecessors. Both back edges route through the incrementing header, so the + * loop is bounded and verifies. Repeated idoms_intersect() at the header must + * stay stable and keep idom(1) at the pre-header (insn 0). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r6 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r6 += 1") +__msg("{{.*}} 1 2: {{.*}} (25) if r6 > 0xa goto pc+3") +__msg("{{.*}} 2 3: {{.*}} (bf) r7 = r6") +__msg("{{.*}} 3 4: {{.*}} (a5) if r7 < 0x5 goto pc-4") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc-5") +__msg("{{.*}} 2 6: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void multi_backedge_header(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r6 += 1; \ + if r6 > 10 goto 2f; \ + r7 = r6; \ + if r7 < 5 goto 1b; \ + goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Irreducible CFG (loop with two entries, insn 3 and insn 5, reached from the + * insn 2 branch). Dominators remain well-defined; this is a convergence test + * for the fixpoint under irreducibility. Note insn 6 is dominated by the branch + * (insn 2), not by either arm. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 4: {{.*}} (05) goto pc+1") +__msg("{{.*}} 2 5: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 2 6: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Idoms are computed per subprog. Layout (libbpf preorder) is: + * main: 0..1, sub: 2..7 + * The callee entry (insn 2) must have idom -1 (subprog reset), and idoms inside + * the callee must reference only the callee's instructions, never main's. The + * in-callee merge (insn 7) is dominated by the in-callee branch (insn 3). The + * branch is on a prandom value so it is not constant-folded away. + */ +static __naked __noinline __used +unsigned long idoms_diamond_sub(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 1; \ + goto 2f; \ +1: r0 = 2; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call pc+1") +__msg("{{.*}} 0 1: {{.*}} (95) exit") +__msg("{{.*}} -1 2: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 2 3: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg("{{.*}} 3 4: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc+1") +__msg("{{.*}} 3 6: {{.*}} (b7) r0 = 2") +__msg("{{.*}} 3 7: {{.*}} (95) exit") +__naked void multi_subprog(void) +{ + asm volatile (" \ + call idoms_diamond_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The first instruction of a subprog (insn 3) is itself a loop header, i.e. it + * has an incoming back edge. Its idom must still be -1 (the subprog entry has no + * dominator). Exercises the idoms[start]=start ... idoms[start]=-1 handling in + * compute_subprog_idoms(). + */ +static __naked __noinline __used +unsigned long idoms_entry_header_sub(void) +{ + asm volatile (" \ +1: r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 0 1: {{.*}} (85) call pc+1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("{{.*}} -1 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void entry_is_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + call idoms_entry_header_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Self-loop: insn 1 is its own predecessor. idom(1) must be the pre-header + * (insn 0); the self-edge is ignored (idoms[pred] == -1 on the first pass, then + * idoms_intersect(a == b) short-circuits). Program is rejected later for an + * infinite loop, but the dump (and idoms) are printed before that. + */ +SEC("socket") +__failure +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (a5) if r0 < 0xa goto pc-1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("infinite loop detected") +__naked void self_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +/* + * Indirect jump (gotox) with a two-entry jump table. The gotox (insn 4) has two + * successors (insn 5 and insn 7), so both targets have the gotox as their only + * predecessor and idom. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (18) r0 = {{0x[0-9a-f]+}}") +__msg("{{.*}} 0 2: {{.*}} (07) r0 += 8") +__msg("{{.*}} 2 3: {{.*}} (79) r0 = *(u64 *)(r0 +0)") +__msg("{{.*}} 3 4: {{.*}} (0d) gotox r0") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__msg("{{.*}} 4 7: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void gotox_jump_table(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad ret0_%= - socket; \ + .quad ret1_%= - socket; \ + .size jt0_%=, 16; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 += 8; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +ret0_%=: \ + r0 = 0; \ + exit; \ +ret1_%=: \ + r0 = 1; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL"; From 8b93e9f92c16603d4df321cb1edb86a2137ab5eb Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 03:48:09 -0700 Subject: [PATCH 439/660] selftests/bpf: Tests for SCEV analysis and loop widening - Computed expression shapes: register and spilled-counter recurrences, substitution at latches, agreeing and conflicting backedge updates, and ALU, extension, byte-swap, load/store and atomic expressions. - Iteration bounds and header-visit counts for: - pre-condition loops using JEQ, JLT, JLE, JGE and JGT; - post-condition loops using JLT, JLE, JGE, JGT and JNE; - signed pre- and post-conditions using JSLT, JSLE, JSGT and JSGE; - increasing and decreasing counters, nonzero initial values, immediate and invariant-register bounds, and immediate exits; - additional exits that reduce the minimum iteration count or tighten the upper bound; - rounded bounds, counter wraparound, count overflow, extreme steps and JNE divisibility. - Widening, backedge clamping and state-pruning convergence, including multiple induction variables used for map-value accesses and nonconstant entry values requiring a common alignment. Check precision propagation to initial values and bounds. - Nested and sibling loops, shared headers, exits to another loop header, exits across multiple nesting levels, and side entries into loop nests. Check conservative summaries for irreducible children while still widening supported reducible descendants, value preservation on inner-loop exits, and detection of nonterminating outer loops. - Invalidation of stack-slot expressions after indirect stack writes and global calls. Widening across bpf_probe_read_kernel() calls in static subprograms. - Stack spill widths and slot states, including narrow spills, zero, unallocated and STACK_MISC slots, and counters spilled and filled using store-release/load-acquire. - Avoid widening registers used to address stack spills, fills and dynptr constructor arguments, including dependencies through nested loops. Conversely, allow widening addresses of byte-sized stores. - Conditional assignments joining scalar values and stack pointers, including preservation of a common base and gcd-derived step. - Pointer widening and comparisons with compatible memory origins, fallback for incompatible origins and nullable pointers, and invalidation of stale packet pointer relationships. - Conservative fallback for unsupported recurrences, unavailable counter spills and zero computed header counts. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-51-bcba9bc53ac9@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../selftests/bpf/progs/verifier_scev.c | 3180 +++++++++++++++++ 2 files changed, 3182 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_scev.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index b45916e04fef..c69c0f7e9cc8 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -108,6 +108,7 @@ #include "verifier_ringbuf.skel.h" #include "verifier_runtime_jit.skel.h" #include "verifier_scalar_ids.skel.h" +#include "verifier_scev.skel.h" #include "verifier_sdiv.skel.h" #include "verifier_search_pruning.skel.h" #include "verifier_sock.skel.h" @@ -296,6 +297,7 @@ void test_verifier_regalloc(void) { RUN(verifier_regalloc); } void test_verifier_ringbuf(void) { RUN(verifier_ringbuf); } void test_verifier_runtime_jit(void) { RUN(verifier_runtime_jit); } void test_verifier_scalar_ids(void) { RUN(verifier_scalar_ids); } +void test_verifier_scev(void) { RUN(verifier_scev); } void test_verifier_sdiv(void) { RUN(verifier_sdiv); } void test_verifier_search_pruning(void) { RUN(verifier_search_pruning); } void test_verifier_sock(void) { RUN(verifier_sock); } diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c new file mode 100644 index 000000000000..59c4b2864db0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -0,0 +1,3180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +#define COND_TEST(name, _initial, _step, _bound, msg, body) \ +SEC("xdp") \ +__success \ +__log_level(2) \ +__flag(BPF_F_TEST_STATE_FREQ) \ +__msg("loop header at {{[0-9]+}}, " msg) \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r6 = %[initial] ll;" \ + "r7 = %[bound] ll;" \ + "r9 = 0;" \ + body \ + "2: r0 = 0;" \ + "exit;" \ + : \ + : __imm_const(initial, _initial), \ + __imm_const(step, _step), \ + __imm_const(bound, _bound) \ + : __clobber_all); \ +} + +/* Optional assembly runs before the tested condition, preserving its latch. */ +#define PRE__COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "if r6 " op " r7 goto 2f;" \ + "r6 += r8;" \ + "goto 1b;") + +#define POST_COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "r6 += r8;" \ + "if r6 " op " r7 goto 1b;") + +/* + * Bound fallback verification of otherwise infinite or extremely long loops. + * JSET is not a linear condition, SCEV ignores it and the tested condition + * alone decides the iteration count. + */ +#define LIMIT_ITERATIONS "r9 += 1; if r9 & 8 goto 2f;" + +#define PRE__COND_TEST2(name, op, initial, step, bound, msg) \ + PRE__COND_TEST(name, op, initial, step, bound, msg, LIMIT_ITERATIONS) + +#define ITER_U64_MAX (~0ULL) +#define ITER_S64_MAX ((1ULL << 63) - 1) +#define ITER_S64_MIN (1ULL << 63) +#define ITER_U32_MAX ((1ULL << 32) - 1) + +struct map_val { + char foo[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); +} map SEC(".maps"); + +typeof(map) other_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, __u64); +} small_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); + __array(values, struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(map_flags, BPF_F_INNER_MAP); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); + }); +} outer_map SEC(".maps"); + +struct bpf_iter__bpf_map_elem { + struct bpf_iter_meta *meta; + struct bpf_map *map; + void *key; + void *value; +}; + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__naked void simple_loop1(void) +{ + asm volatile (" \ + r0 = 0; \ +loop_%=: \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> ?") +__naked void indirect_write_invalidates_scev(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = r10; \ + r1 += -8; \ + *(u64 *)(r1 + 0) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Even a no-op global callee can be freplaced with one that writes to *p. */ +__noinline int global_noop_counter(__u64 *p) +{ + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> ?") +__naked void global_call_invalidates_scev(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 8) = r0;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r0 += 1;" + "*(u64 *)(r10 - 8) = r0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_counter;" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg_next(" scev at latch 3:") +__msg_next(" r0=*fp-8 / (linear *fp-8 1)") +__msg_next(" *fp-8=*fp-8 / (linear *fp-8 1)") +__naked void simple_loop2(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" r1=(+ r1 1) / (linear r1 1)") +__naked void meet_agrees(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ +1: \ + if r1 == 2 goto 3f; \ + if r0 == 7 goto 2f; \ + r1 += 1; \ + goto 1b; \ +2: \ + r1 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__naked void meet_disagrees(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 7 goto 1b; \ + r6 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Both entry state values for r1 should be accounted for */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg("loop header at 2, widening r0 to 0..10 step 1") +__msg_next("loop header at 2, widening r1 to 7..42 step 1") +__naked void entry_state_join(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 7; \ +1: if r0 == 10 goto 2f; /* r1 in [7,42] */ \ + r0 += 1; \ + r1 = 42; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(bswap32 (bswap32 (zext32 (- (- (zext32 (+ (>>...) 1))))))) / ?") +__naked void expr_chain(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r6 <<= 32; \ + r6 >>= 32; \ + w6 += 1; \ + r6 = -r6; \ + w6 = -w6; \ + r6 = bswap32 r6; \ + r6 = bswap32 r6; \ + goto 1b; \ +2: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#define ALU_OP(insn) "r1 = r6; " insn "; r0 += r1;" + +SEC("xdp") +__success +__log_level(2) +__msg("r1 += 2 {{.*}}; r1 r6 -> (+ r6 2)") +__msg("r1 += r7 {{.*}}; r1 r6 -> (+ r6 r7)") +__msg("w1 += 2 {{.*}}; r1 r6 -> (zext32 (+ r6 2))") +__msg("w1 += w7 {{.*}}; r1 r6 -> (zext32 (+ r6 r7))") +__msg("r1 -= 2 {{.*}}; r1 r6 -> (- r6 2)") +__msg("r1 -= r7 {{.*}}; r1 r6 -> (- r6 r7)") +__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (- r6 2))") +__msg("w1 -= w7 {{.*}}; r1 r6 -> (zext32 (- r6 r7))") +__msg("r1 *= 2 {{.*}}; r1 r6 -> (* r6 2)") +__msg("r1 *= r7 {{.*}}; r1 r6 -> (* r6 r7)") +__msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))") +__msg("w1 *= w7 {{.*}}; r1 r6 -> (zext32 (* r6 r7))") +__msg("r1 /= 2 {{.*}}; r1 r6 -> (/ r6 2)") +__msg("r1 /= r7 {{.*}}; r1 r6 -> (/ r6 r7)") +__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ r6 2))") +__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ r6 r7))") +__msg("r1 s/= 2 {{.*}}; r1 r6 -> (s/ r6 2)") +__msg("r1 s/= r7 {{.*}}; r1 r6 -> (s/ r6 r7)") +__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ r6 2))") +__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ r6 r7))") +__msg("r1 %= 2 {{.*}}; r1 r6 -> (% r6 2)") +__msg("r1 %= r7 {{.*}}; r1 r6 -> (% r6 r7)") +__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% r6 2))") +__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% r6 r7))") +__msg("r1 s%= 2 {{.*}}; r1 r6 -> (s% r6 2)") +__msg("r1 s%= r7 {{.*}}; r1 r6 -> (s% r6 r7)") +__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% r6 2))") +__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% r6 r7))") +__msg("r1 |= 2 {{.*}}; r1 r6 -> (| r6 2)") +__msg("r1 |= r7 {{.*}}; r1 r6 -> (| r6 r7)") +__msg("w1 |= 2 {{.*}}; r1 r6 -> (zext32 (| r6 2))") +__msg("w1 |= w7 {{.*}}; r1 r6 -> (zext32 (| r6 r7))") +__msg("r1 &= 2 {{.*}}; r1 r6 -> (& r6 2)") +__msg("r1 &= r7 {{.*}}; r1 r6 -> (& r6 r7)") +__msg("w1 &= 2 {{.*}}; r1 r6 -> (zext32 (& r6 2))") +__msg("w1 &= w7 {{.*}}; r1 r6 -> (zext32 (& r6 r7))") +__msg("r1 ^= 2 {{.*}}; r1 r6 -> (^ r6 2)") +__msg("r1 ^= r7 {{.*}}; r1 r6 -> (^ r6 r7)") +__msg("w1 ^= 2 {{.*}}; r1 r6 -> (zext32 (^ r6 2))") +__msg("w1 ^= w7 {{.*}}; r1 r6 -> (zext32 (^ r6 r7))") +__msg("r1 <<= 2 {{.*}}; r1 r6 -> (<< r6 2)") +__msg("r1 <<= r7 {{.*}}; r1 r6 -> (<< r6 r7)") +__msg("w1 <<= 2 {{.*}}; r1 r6 -> (zext32 (<< r6 2))") +__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 r7))") +__msg("r1 >>= 2 {{.*}}; r1 r6 -> (>> r6 2)") +__msg("r1 >>= r7 {{.*}}; r1 r6 -> (>> r6 r7)") +__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> r6 2))") +__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> r6 r7))") +__msg("r1 s>>= 2 {{.*}}; r1 r6 -> (s>> r6 2)") +__msg("r1 s>>= r7 {{.*}}; r1 r6 -> (s>> r6 r7)") +__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> r6 2))") +__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> r6 r7))") +__msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)") +__msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))") +__msg("r1 = 42 {{.*}}; r1 r6 -> 42") +__msg("w1 = 42 {{.*}}; r1 r6 -> (zext32 42)") +__msg("r1 = r7 {{.*}}; r1 r6 -> r7") +__msg("w1 = w7 {{.*}}; r1 r6 -> (zext32 r7)") +__msg("r1 = (s8)r6 {{.*}}; r1 r6 -> (sext8 r6)") +__msg("r1 = (s16)r6 {{.*}}; r1 r6 -> (sext16 r6)") +__msg("r1 = (s32)r6 {{.*}}; r1 r6 -> (sext32 r6)") +__msg("w1 = (s8)w6 {{.*}}; r1 r6 -> (zext32 (sext8 r6))") +__msg("w1 = (s16)w6 {{.*}}; r1 r6 -> (zext32 (sext16 r6))") +__msg("r1 = bswap16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = bswap32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = bswap64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = be64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#else +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = le64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +#endif +__naked void alu_ops_exprs(void) +{ + asm volatile ( + "r6 = 7;" + "r7 = 2;" + "r8 = 0;" +"1:" + "r0 = 0;" + ALU_OP("r1 += 2") + ALU_OP("r1 += r7") + ALU_OP("w1 += 2") + ALU_OP("w1 += w7") + ALU_OP("r1 -= 2") + ALU_OP("r1 -= r7") + ALU_OP("w1 -= 2") + ALU_OP("w1 -= w7") + ALU_OP("r1 *= 2") + ALU_OP("r1 *= r7") + ALU_OP("w1 *= 2") + ALU_OP("w1 *= w7") + ALU_OP("r1 /= 2") + ALU_OP("r1 /= r7") + ALU_OP("w1 /= 2") + ALU_OP("w1 /= w7") + ALU_OP("r1 s/= 2") + ALU_OP("r1 s/= r7") + ALU_OP("w1 s/= 2") + ALU_OP("w1 s/= w7") + ALU_OP("r1 %%= 2") + ALU_OP("r1 %%= r7") + ALU_OP("w1 %%= 2") + ALU_OP("w1 %%= w7") + ALU_OP("r1 s%%= 2") + ALU_OP("r1 s%%= r7") + ALU_OP("w1 s%%= 2") + ALU_OP("w1 s%%= w7") + ALU_OP("r1 |= 2") + ALU_OP("r1 |= r7") + ALU_OP("w1 |= 2") + ALU_OP("w1 |= w7") + ALU_OP("r1 &= 2") + ALU_OP("r1 &= r7") + ALU_OP("w1 &= 2") + ALU_OP("w1 &= w7") + ALU_OP("r1 ^= 2") + ALU_OP("r1 ^= r7") + ALU_OP("w1 ^= 2") + ALU_OP("w1 ^= w7") + ALU_OP("r1 <<= 2") + ALU_OP("r1 <<= r7") + ALU_OP("w1 <<= 2") + ALU_OP("w1 <<= w7") + ALU_OP("r1 >>= 2") + ALU_OP("r1 >>= r7") + ALU_OP("w1 >>= 2") + ALU_OP("w1 >>= w7") + ALU_OP("r1 s>>= 2") + ALU_OP("r1 s>>= r7") + ALU_OP("w1 s>>= 2") + ALU_OP("w1 s>>= w7") + ALU_OP("r1 = -r1") + ALU_OP("w1 = -w1") + ALU_OP("r1 = 42") + ALU_OP("w1 = 42") + ALU_OP("r1 = r7") + ALU_OP("w1 = w7") + ALU_OP("r1 = (s8)r6") + ALU_OP("r1 = (s16)r6") + ALU_OP("r1 = (s32)r6") + ALU_OP("w1 = (s8)w6") + ALU_OP("w1 = (s16)w6") + ALU_OP("r1 = bswap16 r1") + ALU_OP("r1 = bswap32 r1") + ALU_OP("r1 = bswap64 r1") + ALU_OP("r1 = le16 r1") + ALU_OP("r1 = le32 r1") + /* + * Expressions unchanged by transfer() do not appear in the log. + * Add r1 += 1 to touch the expression and make it appear in the log. + */ + ALU_OP("r1 = le64 r1; r1 += 1") + ALU_OP("r1 = be16 r1") + ALU_OP("r1 = be32 r1") + ALU_OP("r1 = be64 r1; r1 += 1") + "r8 += 1;" + "if r8 < 2 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +#undef ALU_OP + +/* Reset operands and consume results to keep transfer changes live. */ +#define ST_OP(stmt) \ + "r0 = 0; r1 = 1; r2 = 2;" \ + "*(u64 *)(r10 - 8) = -8;" \ + stmt ";" \ + "r0 += r1; r0 += r2;" \ + "r3 = *(u64 *)(r10 - 8); r0 += r3;" + +SEC("xdp") +__success +__log_level(2) +__msg("*(u64 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> 1") +__msg("*(u32 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("*(u16 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("*(u8 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("r1 = *(u64 *)(r10 -8) {{.*}}; r1 1 -> -8") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (zext32 -8)") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (sext32 -8)") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (sext16 -8)") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (sext8 -8)") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> (zext32 -8)") +#else +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> _") +#endif +__msg("r2 = load_acquire((u64 *)(r10 -8)) {{.*}}; r2 2 -> -8") +__msg("store_release((u8 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("store_release((u16 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("store_release((u32 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("store_release((u64 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> 1") +__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> _") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> _, *fp-8 -8 -> _") +__msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42") +__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> _") +__naked void store_load_exprs(void) +{ + asm volatile ( + "r7 = 0;" +"1:" + ST_OP("*(u64 *)(r10 - 8) = r1") + ST_OP("*(u32 *)(r10 - 8) = r1") + ST_OP("*(u16 *)(r10 - 8) = r1") + ST_OP("*(u8 *)(r10 - 8) = r1") + + ST_OP("r1 = *(u64 *)(r10 - 8)") + ST_OP("r1 = *(u32 *)(r10 - 8)") + ST_OP("r1 = *(u16 *)(r10 - 8)") + ST_OP("r1 = *(u8 *)(r10 - 8)") + ST_OP("r1 = *(s32 *)(r10 - 8)") + ST_OP("r1 = *(s16 *)(r10 - 8)") + ST_OP("r1 = *(s8 *)(r10 - 8)") + + ST_OP(".8byte %[load_acquire8]") + ST_OP(".8byte %[load_acquire16]") + ST_OP(".8byte %[load_acquire32]") + ST_OP(".8byte %[load_acquire64]") + ST_OP(".8byte %[store_release8]") + ST_OP(".8byte %[store_release16]") + ST_OP(".8byte %[store_release32]") + ST_OP(".8byte %[store_release64]") + + ST_OP(".8byte %[atomic_add64]") + ST_OP(".8byte %[atomic_fetch_add64]") + ST_OP(".8byte %[atomic_fetch_add32]") + ST_OP(".8byte %[atomic_xchg64]") + ST_OP(".8byte %[atomic_cmpxchg64]") + + ST_OP("*(u64 *)(r10 - 8) = 42") + ST_OP("*(u8 *)(r10 - 7) = r0") + + "r7 += 1;" + "if r7 < 2 goto 1b;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg64, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(load_acquire8, BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire16, BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire32, BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire64, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)), + __imm_insn(store_release8, BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release16, BPF_ATOMIC_OP(BPF_H, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release32, BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release64, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg64, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8)), + __imm_insn(atomic_fetch_add32, BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +#undef ST_OP + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next(" scev at latch 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__naked void nested_loop1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r7 = 0; \ +3: \ + if r7 == 2 goto 4f; \ + r7 += 1; \ + goto 3b; \ +4: \ + goto 1b; \ +2: \ + r0 = r7; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__msg("loop at 1") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 1 to 7") +__msg_next("loop at 4, nested in 1") +__msg_next(" backedge from 6, latch at 4") +__msg_next(" exit from 4 to 1") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 1, latch not identified") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__log_level(2) +__naked void nested_loop_hdr_backedge1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + r6 += 1; \ + r7 = 0; \ +2: \ + if r7 == 2 goto 1b; \ + r7 += 1; \ + goto 2b; \ +3: \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Outer loop {2, 3, 4, 5} contains inner loop {3, 4, 5}. + * The outer loop's only backedge is 4 -> 2, and its latch is also at 4, + * inside the inner loop. Outer loop's latch index is not computed in + * such a case, as there is no mechanism to build SCEVs for such latches + * at the moment. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 6") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 5, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 6") +__msg("scev at header 2:") +__msg_next(" r6=?") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next("scev at header 3:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 5:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("loop header at 2, latch not identified") +__msg("loop header at 3, header_count is [0..2] ") +__naked void nested_loop_hdr_backedge2(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ +1: r7 += 1; \ +2: r6 += 1; \ + if r7 == 2 goto 1b; \ + if r6 < 2 goto 2b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Logical nest: for (r6 = 0; r6 < 3; r6++) + * for (r7 = 0; r7 < 4; r7++) r8++; + * Both backedges target header 3, so the CFG has one loop. r6 advances only + * on the outer backedge and r7 resets there; neither has a linear SCEV. + * r8 advances on both backedges and retains its linear SCEV. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 3{{$}}") +__msg("scev at header 3:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__msg_next(" r7=(any (+ r7 1) 0) / ?") +__msg_next(" r8=(+ r8 1) / (linear r8 1)") +__msg("loop header at 3, unsupported loop: multiple backedges") +__naked void nested_loops_shared_header(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ +1: r8 += 1; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ + r7 = 0; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = r8; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 5, header_count is 5") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loops_outer_latch_true_branch_to_inner(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; /* outer loop */ \ + r1 = 0; \ + if r0 < 10 goto 2f; /* outer latch true branch jumps into inner loop */ \ + goto 3f; \ +2: r1 += 1; /* inner loop */ \ + if r1 < 5 goto 2b; \ + goto 1b; \ +3: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Exact boundary hits for every continuation opcode. + * + * Example: pre_lt + * + * u64 r6 = 1; + * while (r6 < 7) { // r6 ∈ [1,3,5,7] + * r6 += 2; // r6 ∈ [3,5,7] + * } + */ +/* name op initial step bound expected message */ +PRE__COND_TEST (pre_lt, ">=", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_lt, "<", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_le, ">", 1, 2, 7, "header_count is 5 ") +POST_COND_TEST (post_le, "<=", 1, 2, 7, "header_count is 4 ") +PRE__COND_TEST (pre_gt, "<=", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_gt, ">", 9, -2, 3, "header_count is 3 ") +PRE__COND_TEST (pre_ge, "<", 9, -2, 3, "header_count is 5 ") +POST_COND_TEST (post_ge, ">=", 9, -2, 3, "header_count is 4 ") +PRE__COND_TEST (pre_slt, "s>=", -5, 2, 1, "header_count is 4 ") +POST_COND_TEST (post_slt, "s<", -5, 2, 1, "header_count is 3 ") +PRE__COND_TEST (pre_sle, "s>", -5, 2, 1, "header_count is 5 ") +POST_COND_TEST (post_sle, "s<=", -5, 2, 1, "header_count is 4 ") +PRE__COND_TEST (pre_sgt, "s<=", 5, -2, -1, "header_count is 4 ") +POST_COND_TEST (post_sgt, "s>", 5, -2, -1, "header_count is 3 ") +PRE__COND_TEST (pre_sge, "s<", 5, -2, -1, "header_count is 5 ") +POST_COND_TEST (post_sge, "s>=", 5, -2, -1, "header_count is 4 ") +PRE__COND_TEST (pre_ne, "==", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_ne, "!=", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_ne_down, "==", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_ne_down, "!=", 9, -2, 3, "header_count is 3 ") + +/* + * Bounds between two consecutive counter values. + * + * Example: pre_lt_round + * + * u64 r6 = 1; + * while (r6 < 8) { // r6 ∈ [1,3,5,7,9] + * r6 += 2; // r6 ∈ [3,5,7,9] + * } + */ +PRE__COND_TEST (pre_lt_round, ">=", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_le_round, ">", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_gt_round, "<=", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_ge_round, "<", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_slt_round, "s>=", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sle_round, "s>", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sgt_round, "s<=", 5, -2, -2, "header_count is 5 ") +PRE__COND_TEST (pre_sge_round, "s<", 5, -2, -2, "header_count is 5 ") + +/* + * Equality on the first non-strict comparison still takes a backedge. + * + * Example: pre_le_eq + * + * u64 r6 = 7; + * while (r6 <= 7) { // r6 ∈ [7,9] + * r6 += 2; // r6 ∈ [9] + * } + */ +PRE__COND_TEST (pre_le_eq, ">", 7, 2, 7, "header_count is 2 ") +PRE__COND_TEST (pre_ge_eq, "<", 3, -2, 3, "header_count is 2 ") +PRE__COND_TEST (pre_sle_eq, "s>", 1, 2, 1, "header_count is 2 ") +PRE__COND_TEST (pre_sge_eq, "s<", -1, -2, -1, "header_count is 2 ") + +/* + * Unsigned order across the sign bit. + * + * Example: pre_lt_sign + * + * u64 H = 1ULL << 63, r6 = H - 2; + * while (r6 < H + 1) { // r6 ∈ [H-2,H-1,H,H+1] + * r6 += 1; // r6 ∈ [H-1,H,H+1] + * } + */ +PRE__COND_TEST (pre_lt_sign, ">=", ITER_S64_MAX - 1, 1, ITER_S64_MIN + 1, "header_count is 4 ") +PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S64_MAX - 1, "header_count is 4 ") + +/* + * The first latch comparison is false. + * + * Example: pre_lt_false + * + * u64 r6 = 7; + * while (r6 < 7) { // r6 ∈ [7] + * r6 += 2; // unreachable + * } + */ +PRE__COND_TEST (pre_lt_false, ">=", 7, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_le_false, ">", 8, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_gt_false, "<=", 3, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_ge_false, "<", 2, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_slt_false, "s>=", 1, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sle_false, "s>", 2, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sgt_false, "s<=", -1, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_sge_false, "s<", -2, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_ne_false, "==", 7, 2, 7, "can't compute iterations count") + +/* + * The counter moves away from the exit; bound fallback verification. + * + * Example: pre_lt_dir + * + * u64 r6 = 5, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [5,3,1,U64_MAX] + * r6 -= 2; // r6 ∈ [3,1,U64_MAX] + * } + */ +PRE__COND_TEST2(pre_lt_dir, ">=", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_le_dir, ">", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_dir, "<=", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_dir, "<", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_dir, "s>=", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_dir, "s>", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_dir, "s<=", 1, 2, -1, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2, -1, "can't compute iterations count") + +/* + * The predicted exit would wrap the counter. + * + * Example: pre_lt_wrap + * + * u64 M = U64_MAX, r6 = M - 1, r9 = 0; + * while (++r9 <= 8 && r6 < M) { // r6 ∈ [M-1,0,2,...,14] + * r6 += 2; // r6 ∈ [0,2,...,14] + * } + */ +PRE__COND_TEST2(pre_lt_wrap, ">=", ITER_U64_MAX - 1, 2, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_le_wrap, ">", ITER_U64_MAX, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_wrap, "<=", 1, -2, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_wrap, "<", 0, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_wrap, "s>=", ITER_S64_MAX - 1, 2, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_wrap, "s>", ITER_S64_MAX, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_wrap, "s<=", ITER_S64_MIN + 1, -2, ITER_S64_MIN, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * The non-strict count itself would overflow u64. + * + * Example: pre_le_count_ovf + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 <= U64_MAX) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_le_count_ovf, ">", 0, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_count_ovf, "<", ITER_U64_MAX, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_count_ovf, "s>", ITER_S64_MIN, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * JNE divisibility and equality across unsigned wraparound. + * + * Example: pre_ne_rem + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 != 8) { // r6 ∈ [1,3,...,17] + * r6 += 2; // r6 ∈ [3,5,...,17] + * } + */ +PRE__COND_TEST2(pre_ne_rem, "==", 1, 2, 8, "can't compute iterations count") +PRE__COND_TEST2(pre_ne_rem_down, "==", 9, -2, 2, "can't compute iterations count") +PRE__COND_TEST (pre_ne_wrap_up, "==", ITER_U64_MAX - 1, 2, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_wrap_down, "==", 1, -2, ITER_U64_MAX, "header_count is 2 ") + +/* + * Largest supported header count, sentinel, and u32 overflow. + * + * Example: pre_lt_max_hdrs + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 < U32_MAX - 2) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_lt_max_hdrs, ">=", 0, 1, ITER_U32_MAX - 2, "header_count is [0..4294967294] ") +PRE__COND_TEST2(pre_lt_sentinel, ">=", 0, 1, ITER_U32_MAX - 1, "can't compute iterations count") +PRE__COND_TEST2(pre_lt_hdr_ovf, ">=", 0, 1, ITER_U32_MAX, "can't compute iterations count") + +/* + * The extreme signed step values remain valid unsigned magnitudes. + * + * Example: pre_lt_step_max + * + * u64 r6 = 0; + * while (r6 < S64_MAX) { // r6 ∈ [0,S64_MAX] + * r6 += S64_MAX; // r6 ∈ [S64_MAX] + * } + */ +PRE__COND_TEST (pre_lt_step_max, ">=", 0, ITER_S64_MAX, ITER_S64_MAX, "header_count is 2 ") +PRE__COND_TEST (pre_gt_step_min, "<=", ITER_S64_MIN, ITER_S64_MIN, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, ITER_S64_MIN, "header_count is 2 ") + +/* + * A zero step cannot establish a finite iteration count. + * + * Example: pre_lt_zero_step + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [1], nine header visits + * r6 += 0; // r6 ∈ [1] + * } + */ +PRE__COND_TEST2(pre_lt_zero_step, ">=", 1, 0, 7, "can't compute iterations count") + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10 ") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("processed 5 insns") +__naked void post_cond_jlt(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Explore with r7=10 first, r7=4096 next. + * Should trigger invalid stack read if r7 is marked precise. + * + * r0 = 0; + * r6 = 0; + * r7 = bpf_get_prandom_u32() == 42 ? 10 : 4096; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < r7); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6,r7 stack=:") +__msg("loop header at 6, widening r0 to 0..9 step 1") +__msg("loop header at 6, header_count is 4096 ") +__msg("loop header at 6, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_bound_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = 4096; \ + if r0 != 42 goto 1f; \ + r7 = 10; \ +1: r0 = 0; \ + r6 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < r7 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Explore with r6=4086 first, r6=0 next. + * Should trigger invalid stack read if r6 is marked precise. + * + * r0 = 0; + * r6 = bpf_get_prandom_u32() == 42 ? 4086 : 0; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < 4096); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6 stack=:") +__msg("loop header at 5, widening r0 to 0..9 step 1") +__msg("loop header at 5, header_count is 4096 ") +__msg("loop header at 5, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_base_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + if r0 != 42 goto 1f; \ + r6 = 4086; \ +1: r0 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < 4096 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Spill size should be taken into account when evaluating bound scev. + * + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x200) break; // hidden from scev, limit main pass iterations. + * } while (r0 < r1); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound(void) +{ + asm volatile (" \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x200 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* Even when the saved value fits, the upper bytes of the slot are unknown. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound_fits(void) +{ + asm volatile (" \ + r0 = 10; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* + * r0 = 255; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * do { + * r0 = *(u64 *)(r10 - 8); // wide load, widening should bail out here. + * r0++; + * *(u64 *)(r10 - 8) = r0; + * } while (r0 < 265); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, can't compute iterations count") +__not_msg("widening") +__naked void no_widen_small_spill_counter(void) +{ + asm volatile (" \ + r0 = 255; \ + *(u8 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 265 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Full-width counterpart: the counter can be widened from 5 to 14. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_full_spill_counter(void) +{ + asm volatile (" \ + r0 = 5; \ + *(u64 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 15 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_acquire_release_counter(void) +{ + asm volatile ( + "r0 = 5;" + "*(u64 *)(r10 - 8) = r0;" +"1:" + ".8byte %[load_acquire];" + "r0 += 1;" + ".8byte %[store_release];" + "if r0 < 15 goto 1b;" + "exit;" + : + : __imm_insn(load_acquire, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)), + __imm_insn(store_release, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_0, -8)) + : __clobber_all); +} + +/* + * r6 scev is (any r6 *fp-8). The byte store leaves memory at zero, + * but the spill metadata retains 256. Widen r6 to an unknown scalar + * instead of [1,256], which would exclude the zero loaded from memory. + * + * r0 = 0; + * *(u64 *)(r10 - 8) = r0; + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * r6 = 1; + * r7 = 0; + * do { + * r0 = r6; + * r6 = *(u64 *)(r10 - 8); + * r7++; + * } while (r7 < 2); + * return r0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 6:") +__msg_next(" r6=*fp-8 / (any r6 *fp-8)") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 6, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__naked void widen_any_small_spill_leaf(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r6 = 1; \ + r7 = 0; \ +1: r0 = r6; \ + r6 = *(u64 *)(r10 - 8); \ + r7 += 1; \ + if r7 < 2 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Widening (any *fp-8 7) must allocate the slot before writing its state. + * The unallocated entry value makes the union an unknown scalar. + * + * r6 = 0; + * do { + * if (r6 == 1) + * *(u64 *)(r10 - 8) = 7; + * r0 = *(u64 *)(r10 - 8); + * r6++; + * } while (r6 < 3); + * return r0; + */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 1:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 1, widening r6 to 0..2 step 1") +__msg("2: (7a) *(u64 *)(r10 -8) = 7 ; R10=fp0 fp-8=7 loop_stack=1") +__naked void widen_any_unallocated_slot(void) +{ + asm volatile (" \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A spilled zero entry contributes zero to the union. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("loop header at 2, widening *fp-8 to 0..7 step 1") +__naked void widen_any_zero_spill(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A byte store inside the slot leaves STACK_MISC and an unknown scalar value. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 2:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 2, widening r6 to 0..2 step 1") +__msg("3: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar() R10=fp0 fp-8=m???????") +__naked void widen_any_misc_slot(void) +{ + asm volatile (" \ + *(u8 *)(r10 - 1) = 1; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + r1 = *(u64 *)(r10 - 8); \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int probe_read_stack(void) +{ + asm volatile ( + "r2 = 8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "exit;" + : + : __imm(bpf_probe_read_kernel) + : __clobber_all); +} + +/* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ +SEC("tracepoint") +__success +__log_level(2) +__msg("scev at header 4:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" *fp-8=?") +__msg("loop header at 4, widening r6 to 0..9 step 1") +__naked void widen_subprog_probe_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 += 1;" + "if r6 < 10 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* An induction variable whose range overflows s64 is widened to the full range. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 4, header_count is 10") +__msg("loop header at 4, widening r1 to 0..9 step 1") +__msg("loop header at 4, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void widen_overflowing_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 |= 1; /* not an unknown scalar, those need no widening */ \ + r1 = 0; \ +1: r6 += 24; \ + r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The bound slot is unallocated at the first loop header. SCEV must + * treat it as an unknown scalar. JSET limits main-pass iterations. + * + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x80) break; + * } while (r0 < r1); + * return 0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, can't compute iterations count") +__not_msg("widening") +__naked void eval_expr_unallocated_stack_bound(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, header_count is 8") +__msg("loop header at 3, widening r0 to 2..9 step 1") +__msg("3: R0=scalar(smin=umin=smin32=umin32=2,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jlt_with_base(void) +{ + asm volatile (" \ + r7 = 10 ll; /* ldimm64 for a twist */ \ + r0 = 2; \ +1: r0 += 1; \ + if r0 < r7 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 4 ") +__naked void latch_base_differs_from_step(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r1 = r0; \ + r1 += 1; \ + if r1 >= 10 goto 2f; \ + r0 += 4; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 5 insns") +__naked void post_cond_jle(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 <= 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 >= 10 goto 2f; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 6 insns") +__naked void pre_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 >= 10 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("1: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("2: (55) if r0 != 0x3 goto pc-2") +__msg("3: (95) exit") +__msg("loop header at 1, clamping r0") +__msg("from 2 to 1: safe") +__msg("processed 5 insns") +__naked void post_cond_jne(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 != 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 -1) / (linear r0 -1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") +__msg("loop header at 1, header_count is 3 ") +__msg("loop header at 1, widening r0 to 1..3 step 1") +__msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") +__msg("loop header at 1, clamping r0 to 1..2 step 1") +__msg("processed 5 insns") +__naked void post_cond_jne_neg_step(void) +{ + asm volatile (" \ + r0 = 3; \ +1: r0 += -1; \ + if r0 != 0 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}})") +__msg("1: (15) if r0 == 0x3 goto pc+2 ; R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("2: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("3: (05) goto pc-3") +__msg("loop header at 1, clamping r0") +__msg("1: safe") +__msg("from 1 to 4: R0=3") +__msg("4: R0=3") +__msg("4: (95) exit") +__msg("processed 6 insns") +__naked void pre_cond_je1(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 == 3 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is [0..3] ") +__naked void one_backedge_two_exits(void) +{ + asm volatile (" \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 0 goto 2f; \ + r6 += 1; \ + if r6 != 3 goto 1b; \ +2: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 11:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=(+ r1 2) / (linear r1 2)") +__msg_next(" scev at latch 16:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg_next(" r1=(+ r1 2) / (linear (+ r1 2) 2)") +__msg("loop header at 11, widening r0") +__msg("loop header at 11, widening r1") +__msg("11: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=7,var_off=(0x0; 0x7)) R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=14,var_off=(0x0; 0xe),step=0+2)") +/* loop exit */ +__msg("16: (a5) if r0 < 0x8 goto pc-6") +__msg("exiting loop 11") +__msg("17: (95) exit") +/* second iteration */ +__msg("loop header at 11, clamping r0") +__msg("loop header at 11, clamping r1") +/* iteration convergence */ +__msg("from 16 to 11: safe") +__not_msg("{{^}}11:") +/* map lookup error path */ +__msg("from 7 to 17: safe") +__naked void correlated_regs(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r0 = 0; \ + r1 = 0; \ +1: r2 = r6; \ + r2 += r1; \ + *(u8 *)(r2 + 0) = 1; \ + r0 += 1; \ + r1 += 2; \ + if r0 < 8 goto 1b; \ +2: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * k = 0 + * for (i = 0; i < 4; i++): + * for (j = 0; j < 4; j++): + * k += 1 + * k <<= 1 // make SCEV construction not possible + * k >>= 1 + * map[k] = 1 // make k precise + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("19: (72) *(u8 *)(r4 +0) = 1 ; R4=map_value(id={{.*}},map=map,ks=4,vs=1024,imm=16)") +__not_msg("19: ") +__msg("processed 106 insns") +__naked void nested_loops_precise_var1(void) +{ + asm volatile (" \ + *(u64*)(r10 - 8) = 0; \ + r1 = %[map] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r1 = 0; \ + r3 = 0; \ + /* outer loop */ \ +1: r2 = 0; \ + /* inner loop */ \ +2: r2 += 1; \ + r3 += 1; \ + r3 <<= 1; \ + r3 >>= 1; \ + if r2 < 4 goto 2b; \ + r1 += 1; \ + if r1 < 4 goto 1b; \ + r4 = r0; \ + r4 += r3; \ + *(u8 *)(r4 + 0) = 1; \ + r0 = 0; \ +3: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 2, header_count is 100 ") +__msg("loop header at 4, header_count is [0..100] ") +__msg("loop header at 6, header_count is [0..100] ") +__msg("processed 20 insns") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loop_with_two_exits(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ +1: r6 += 1; \ + r7 = 0; \ +2: r7 += 1; \ + r8 = 0; \ +3: r8 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto +1; \ + goto 4f; \ + if r8 < 100 goto 3b; \ + if r7 < 100 goto 2b; \ +4: if r6 < 100 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__naked void exit_loop_into_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ +loop_a_%=: \ + r1 += 1; \ + if r1 < 10 goto loop_a_%=; \ +loop_b_%=: \ + r2 += 1; \ + if r2 < 10 goto loop_b_%=; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * This exercises verifier.c:loop_stack_{pop,push}() implementation, + * at 'goto d' loops 'b' and 'a' have to be popped from stack, + * while loops 'c' and 'd' have to be pushed to stack. + * + * loop a: // header 5 + * loop b: // header 6 + * if (rand) goto d; // 8 -> 12, side entry into inner loop d + * ... + * loop c: // header 11 + * loop d: // header 12 + * ... + */ +SEC("xdp") +__log_level(2) +__msg("loop at 5") +__msg("loop at 6, nested in 5") +__msg("loop at 11, irreducible") +__msg("loop at 12, nested in 11") +/* entry via if r0 == 5 goto d_%= false branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("12: R8=1 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) loop_stack=11,12") +/* entry via if r0 == 5 goto d_%= true branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("from 8 to 12: R8=0 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) R10=fp0 loop_stack=11,12") +__naked void enter_nested_loop_from_side(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ + r9 = 0; \ +a_%=: r6 += 1; \ +b_%=: r7 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 5 goto d_%=; \ + if r7 < 3 goto b_%=; \ + if r6 < 3 goto a_%=; \ +c_%=: r8 += 1; \ +d_%=: r9 += 1; \ + if r9 < 3 goto d_%=; \ + if r8 < 3 goto c_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B (header 5), + * which contains C (counter in r9). + * Entering B at 'body' saves and restores r6, making it appear invariant. + * Entering B at 'alternate' skips the save and modifies r6 instead. + * Hence A must not infer a SCEV expression for r6. + * SCEV expression for r9 in C should still be computed. + * + * 0: r6 = 0; + * do { // A + * 1: r6++; + * 2: r0 = bpf_get_prandom_u32(); + * 3: r7 = 0; + * 4: if (r0 > 5) goto alternate; + * 5: B: r8 = r6; // B + * 6: goto body; + * 7: alternate: + * r8 = r6; + * 8: r8++; + * 9: body: + * r6 = r8; + * 10: r9 = 0; + * do { // C + * 11: r9++; + * 12: } while (r9 < 3); + * 13: r7++; + * 14: if (r7 < 4) goto B; + * 15: } while (r6 < 4); + * 16: r0 = 0; + * 17: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 5, nested in 1, irreducible") +__msg("loop at 11, nested in 5") +__msg("scev at header 1:") +__msg_next(" r6=?") +__msg("scev at header 11:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg("loop header at 11, widening r9 to 0..2 step 1") +__not_msg("loop header at 1, widening r6") +__naked void nested_irreducible_loop(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r8 = r6; \ + goto body_%=; \ +alternate_%=: \ + r8 = r6; \ + r8 += 1; \ +body_%=: \ + r6 = r8; \ + r9 = 0; \ +c_%=: r9 += 1; \ + if r9 < 3 goto c_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B. + * Loop B has an exit edge from A. This is a corner case + * observed during testing: insn_aux_data[l1].need_scev == true, + * but SCEV environment is not constructed. + * + * r6 = 0; + * do { // A + * r6++; + * r0 = bpf_get_prandom_u32(); + * r7 = 0; + * if (r0 > 5) goto alternate; + * do { // B + * r7++; + * alt: r0 = bpf_get_prandom_u32(); + * l1: if (r0 == 42) + * goto exit; + * r7++; + * } while (r7 < 4); + * } while (r6 < 4); + * r0 = 0; + * exit: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__naked void nested_irreducible_loop2(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r7 += 1; \ +alternate_%=: \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto exit_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ +exit_%=:exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Induction variable seeded from a non-constant value. r7 enters the loop as a + * range aligned to 2 (prandom & 0x6 -> {0,2,4,6}) and is incremented by a + * non-power-of-2 slope of 6. Since the entry value is not a single point, only + * the power-of-two alignment shared by the entry value and the slope can be + * guaranteed, so the widened step is 2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 4, widening r7 to 0..18 step 2") +__msg("R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=18,var_off=(0x0; 0x1e),step=0+2)") +__naked void widen_nonconst_base(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0x6; \ + r6 = 0; \ +1: r7 += 6; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * After excluding zero from {0,4,8,12}, the interval starts at 1 while the + * values remain multiples of 4. Widening over three iterations must preserve + * base 0 and include 20. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 3 ") +__msg("5: R7=scalar({{.*}}smax=umax=smax32=umax32=20,{{.*}}step=0+4)") +__naked void widen_nonconst_base_refined(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0xc; \ + if r7 < 1 goto 2f; \ + r8 = 0; \ +1: r7 += 4; \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Two nested loops with (1) exiting directly to (2): + * + * for (r6 = 0; r6 < 4; r6++) { + * r7 = 0; + * for (; r7 < 3; r7++) {} // (1) + * for (; r7 != 0; r7--) {} // (2) + * } + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 1, widening r6 to 0..3 step 1") +__msg("loop header at 2, widening r7 to 0..2 step 1") +__msg("exiting loop 2") +__msg("entering loop 4") +__msg("loop header at 4, widening r7 to 1..3 step 1") +__naked void sibling_inner_loops(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r7 = 0; \ +2: r7 += 1; \ + if r7 < 3 goto 2b; \ +3: r7 += -1; \ + if r7 != 0 goto 3b; \ + r6 += 1; \ + if r6 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 0, can't compute iterations count") +__naked void uninit_slot_counter(void) +{ + asm volatile (" \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The assignment happens only on the second iteration: + * + * r7 = 5; + * for (r6 = 0; r6 < 3; r6++) + * if (r6 == 1) + * r7 = 10; + * if (r7 != 10) + * invalid_stack_read(); + * + * R7 is always 10 at the real exit. Widening loses the correlation between + * R6 and R7, leaving R7 in [5, 10] both at the loop header and after the loop. + * The verifier therefore rejects the possible invalid stack read. + */ +SEC("xdp") +__failure +__log_level(2) +__msg("2: {{.*}}R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("7: R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("invalid read from stack R10 off=0 size=8") +__naked void conditional_assignment_on_second_iteration(void) +{ + asm volatile (" \ + r7 = 5; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = 10; \ +2: r6 += 1; \ + goto 1b; \ +3: if r7 == 10 goto 4f; \ + r0 = *(u64 *)(r10 + 0); \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The latch bound has header SCEV (any r7 100), so it is not invariant: + * + * u64 i = 0, n = 3; + * for (;;) { + * ++i; + * if (i >= n) + * break; + * if (i == 1) + * n = 100; + * } + * + * The ANY expression must also appear at the latch, rather than a bare r7 + * that would incorrectly be treated as a loop-invariant bound. + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=(any r7 100) / (any r7 100)") +__msg_next(" scev at latch 3:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg_next(" r7=r7 / (any r7 100)") +__naked void no_widen_changing_latch_bound(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 3; \ +loop_%=: \ + r6 += 1; \ + if r6 >= r7 goto exit_%=; \ + if r6 != 1 goto next_%=; \ + r7 = 100; \ +next_%=: \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Join two stack pointer offsets via (any r7 r8), with r8 loop-invariant. + * The joined range [-16, -8] still points to initialized stack memory, + * so dereferencing r7 after the loop is safe. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at 7, widening r7 to -16..-8 step 1") +__msg("7: {{.*}}R7=fp(smin=smin32=-16,smax=smax32=-8,") +__msg("12: R7=fp(smin=smin32=-16,smax=smax32=-8,") +__naked void conditional_stack_pointer_assignment(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + *(u64 *)(r10 - 8) = 0; \ + r7 = r10; \ + r7 += -16; \ + r8 = r10; \ + r8 += -8; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = *(u64 *)(r7 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Both choices are valid, but combining their offsets while retaining + * small_map's metadata would invent small_map + 32, outside its value. + * + * r6 = lookup(small_map, 0); + * r7 = lookup(map, 0) + 32; + * for (r8 = 0; r8 < 3; r8++) { + * if (r8 == 1) + * r6 = r7; + * r0 = *(u32 *)(r6 + 0); + * } + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_map_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r7 = r0; \ + r7 += 32; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r0 = *(u32 *)(r6 + 0); \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(small_map), __imm_addr(map) + : __clobber_all); +} + +__naked __noinline __used static int any_stack_origin_callee(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = r1; \ + r7 = r10; \ + r7 += -8; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r8 += 1; \ + if r8 < 3 goto 1b; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The callee chooses between caller fp-16 and its own fp-8: + * + * r6 = r1; + * r7 = fp - 8; + * for (r8 = 0; r8 < 3; r8++) + * if (r8 == 1) + * r6 = r7; + * return *(u64 *)r6; + * + * Both locations hold zero. Their offsets belong to different frames + * and cannot be combined into one stack pointer. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_stack_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + r1 = r10; \ + r1 += -16; \ + call any_stack_origin_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). + * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. + */ +SEC("xdp") +__success __retval(1) +__log_level(2) +__msg("r7 += 1 {{.*}}step=1+6)") +__msg("r8 += 1 {{.*}}step=1+9)") +__msg("loop header at 9, widening r7 to 1..28 step 3") +__msg("9: {{.*}}R7=scalar({{.*}},step=1+3)") +__msg("14: R7=scalar({{.*}},step=1+3)") +__naked void conditional_scalar_assignment_gcd(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r7 = r0; \ + r7 *= 6; \ + r7 += 1; \ + r8 = r0; \ + r8 *= 9; \ + r8 += 1; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = r7; \ + r0 %%= 3; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r7 = ctx->data; + * r8 = r7 + * for (r6 = 0; r6 < 10 && random() != 42; r7++, r6++); + * if (r7 >= ctx->data_end) + * return; + * *(r8 + 4); // At this point the loop executed unknown number of times. + * // Hence, r7 range gives no information about r8. + */ +SEC("tc") +__failure +__msg("R8 min value is outside of the allowed memory range") +__naked void break_pkt_pointers_id(void) +{ + asm volatile (" \ + r7 = *(u32*)(r1 + %[__sk_buff_data]); \ + r8 = r7; \ + r9 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto 3f; \ + r6 += 1; \ + r7 += 1; \ + if r6 < 10 goto 1b; \ +3: if r7 >= r9 goto 2f; \ + r0 = *(u8*)(r8 + 4); \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * A loop induction variable used to compute the base address of a store to the + * stack must not be widened: the spill offset would become varying, which the + * verifier does not track. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_spill(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} +/* + * Same hazard across a loop nest: the outer induction variable r2 addresses a + * stack store performed inside the inner loop. The dependency is pulled up from + * the inner loop, so the outer loop must not widen r2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__msg("loop header at 3, widening r1 to 0..1 step 1") +__naked void no_widen_stack_spill_nested(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r1 = 0; \ +2: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r1; \ + r1 += 1; \ + if r1 < 2 goto 2b; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop restores r6 on its backedge but exits with r6 = -1000 on break. + * Outer loop must account for the exit value instead of deriving r6 = r6_entry + n. + * On the first outer backedge r6 is -999, so the next map access is invalid. + * + * u8 *r7 = map_value; + * s64 r6 = 0; + * do { + * 1: r9 = 0; + * r0 = r7[r6]; + * while (true) { + * 2: r8 = r6; + * r6 = -1000; + * if (++r9 > 5) + * break; + * r6 = r8; + * 3: } + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 9:") +__msg_next(" r6=(+ ? 1) / ?") +__msg_next(" scev at latch 20:") +__msg_next(" r6=(+ ? 1) / (+ ? 1)") +__msg_next("scev at header 13:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=-1000 / -1000") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__msg("R6=-999") +__msg("R1 min value is negative") +__naked void nested_loop_exit_clobbers_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -8;" + "r1 = %[map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 4f;" + "r7 = r0;" + "r6 = 0;" +"1:" + "r9 = 0;" + "r1 = r7;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" +"2:" + "r8 = r6;" + "r6 = -1000;" + "r9 += 1;" + "if r9 > 5 goto 3f;" + "r6 = r8;" + "goto 2b;" +"3:" + "r6 += 1;" + "if r6 s< 3 goto 1b;" +"4:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * The inner loop exits both nested loops with r6 unchanged, or exits only itself with r6 = -1000. + * The middle loop repairs the latter exit with a stack fill. It therefore preserves r6, + * so the outermost loop must retain r6_entry + n. + * + * r6 = 0; + * do { + * r7 = 0; + * do { + * spill = r6; + * r9 = 0; + * do { + * r8 = r6; + * if (random() & 1) + * goto next; + * r6 = -1000; + * if (++r9 >= 2) + * break; + * r6 = r8; + * } while (true); + * r6 = spill; + * } while (++r7 < 2); + * next: + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__success __retval(3) +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 ? -> r6") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("scev at header 2:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("scev at header 4:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg(" scev at latch 9:") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__naked void nested_loop_exit_preserves_reg(void) +{ + asm volatile ( + "r6 = 0;" +"1:" + "r7 = 0;" +"2:" + "*(u64 *)(r10 - 8) = r6;" + "r9 = 0;" +"3:" + "r8 = r6;" + "call %[bpf_get_prandom_u32];" + "if r0 & 1 goto 5f;" + "r6 = -1000;" + "r9 += 1;" + "if r9 >= 2 goto 4f;" + "r6 = r8;" + "goto 3b;" +"4:" + "r6 = *(u64 *)(r10 - 8);" + "r7 += 1;" + "if r7 < 2 goto 2b;" +"5:" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Complement of no_widen_stack_spill: a sub-register (1-byte) store to the stack + * lands as STACK_MISC and carries no tracked value, so the induction variable + * addressing it (r2) is still widened. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("widening r2 to 0..24 step 8") +__naked void widen_byte_stack_store(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u8 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A fill (BPF_LDX) at a varying stack offset loses precision just like a spill, + * so the induction variable computing the load base (r2) must not be widened. + * The slots are initialized up front so the fill itself is a valid read. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_fill(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 64) = r0; \ + *(u64 *)(r10 - 56) = r0; \ + *(u64 *)(r10 - 48) = r0; \ + *(u64 *)(r10 - 40) = r0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + r4 = *(u64 *)(r3 + 0); \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * struct xdp_md *slots[2] = { ctx, ctx }; + * + * r7 = 0; + * for (r8 = 0; r8 < 2; r8++) { + * value = slots[r7]->data; + * if (i == 0) + * r7 = 1; + * } + * + * SCEV expression for r7 is (any r7 8) and r7 is used to + * address the stack memory. Avoid widening it, otherwise + * verifier won't know the type of the slots[r7] expression. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 4:") +__msg_next("r7=(any r7 8) / (any r7 8)") +__msg("loop header at 4, can't widen r7, expr is {{.*}}, requires exact stack-offset tracking") +__naked void no_widen_stack_fill_any(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + r7 = 0; \ + r8 = 0; \ +1: r2 = r10; \ + r2 += -16; \ + r2 += r7; \ + r3 = *(u64 *)(r2 + 0); /* slots[r7] */ \ + r0 = *(u32 *)(r3 + 0); /* slots[r7]->data */ \ + if r8 != 0 goto 2f; \ + r7 = 8; /* conditionally update r7 */ \ +2: r8 += 1; \ + if r8 < 2 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A dynptr/iter/irq/res_spin_lock call initializes a stack object through a + * pointer argument, which acts like a spill base: the induction variable + * computing that argument's varying stack offset must not be widened, otherwise + * the slot can't be resolved. Here each iteration constructs an xdp dynptr at + * &dptrs[i]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("can't widen {{.*}}, requires exact stack-offset tracking") +#ifndef __clang__ +/* The issue is unrelated to SCEV */ +__skip("GCC emits a stack-pointer loop condition the verifier cannot resolve") +#endif +int no_widen_dynptr_kfunc_arg(struct xdp_md *ctx) +{ + struct bpf_dynptr dptrs[4]; + int i; + +#ifdef __clang__ +#pragma clang loop unroll(disable) +#else +#pragma GCC unroll 0 +#endif + for (i = 0; i < 4; i++) + bpf_dynptr_from_xdp(ctx, 0, &dptrs[i]); + + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to -16..-13 step 1") +__naked void ptr_stack(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int ptr_stack_callee(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r1; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; /* bound main pass iteration, but hide it from SCEV */ \ + if r6 < r7 goto 1b; /* r6 and r7 are from different frames, can't be widened */ \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* The end pointer belongs to the caller's frame. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (fp), r7 (fp)") +__not_msg("widening r6") +__naked void ptr_stack_other_frame(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -12; \ + call ptr_stack_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* Metadata and packet data have different origins. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (pkt_meta), r7 (pkt)") +__not_msg("widening r6") +__naked void ptr_packet_meta_other_origin(void) +{ + asm volatile (" \ + r6 = *(u32 *)(r1 + %[data_meta]); \ + r7 = *(u32 *)(r1 + %[data]); \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r0 = 0; \ + exit; \ +" : + : __imm_const(data, offsetof(struct xdp_md, data)), + __imm_const(data_meta, offsetof(struct xdp_md, data_meta)) + : __clobber_all); +} + +/* Two lookups in a 1-element array count as the same memory origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value_two_lookups(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(small_map) + : __clobber_all); +} + +/* + * For inner array maps with BPF_F_INNER_MAP flag the actual array size + * might differ from metadata in map declaration. Hence, r6 and r7 below + * cannot be directly compared / iterations count can't be inferred. + * + * r8 = bpf_map_lookup_elem(outer_map, ...); // inner map + * r6 = bpf_map_lookup_elem(r8, ...); + * r7 = bpf_map_lookup_elem(r8, ...); + * r7 += 4; + * do { r6++; ... } while (r6 < r7); + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_two_lookups_inner_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[outer_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = r8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r1 = r8; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 2f; /* main pass iteration guard */ \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(outer_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +/* Equal offsets into different maps do not establish a common origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_other_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[other_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map), __imm_addr(other_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_mem(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +/* Separate reservations have different pointer IDs. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (ringbuf_mem), r7 (ringbuf_mem)") +__not_msg("widening r6") +__naked void ptr_mem_other_reservation(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 3f; \ + r8 = r0; \ + r6 = r0; \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r9 = r0; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r1 = r9; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_buf(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* Separate context loads receive different pointer IDs. */ +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (buf), r7 (buf)") +__not_msg("widening r6") +__naked void ptr_buf_other_load(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = *(u64 *)(r1 + %[value]); \ + if r7 == 0 goto 2f; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* A pointer-versus-scalar latch must not establish an iteration count. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r1 (map_value), scalar 4") +__not_msg("widening r1") +__naked void ptr_vs_scalar_no_widen(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto out_%=; \ + r1 = r0; \ + r0 = 0; \ +loop_%=: \ + r1 += 1; \ + r0 += 1; \ + if r0 & 8 goto out_%=; \ + if r1 < 4 goto loop_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* A nullable pointer must not be widened before its null check. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 9, can't widen r1, expr is (+ r1 8)") +__naked void no_widen_maybe_null(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r6 = 0; \ +loop_%=: \ + if r1 == 0 goto exit_%=; \ + r1 += 8; \ + r6 += 1; \ + if r6 < 3 goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * A terminating inner loop does not prove termination of the outer loop. + * + * u64 r7 = 0; + * while (bpf_get_prandom_u32() != 42) { + * r7 = (r7 + 1) & 0xf; + * u64 r8 = 0; + * do { + * r8++; + * } while (r8 < 3); + * } + * return 0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("infinite loop detected at insn 6") +__naked void nested_terminating_prune(void) +{ + asm volatile (" \ + r7 = 0; \ +outer_%=: \ + r7 += 1; \ + r7 &= 0xf; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto out_%=; \ + r8 = 0; \ +inner_%=: \ + r8 += 1; \ + if r8 < 3 goto inner_%=; \ + goto outer_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Returning directly from the inner loop to the outer header changes the + * active stack from [1, 4] to [1]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 1{{$}}") +__msg("loop at 4, nested in 1") +__msg("entering loop 1") +__msg("entering loop 4") +__msg("exiting loop 4") +__not_msg("entering loop 1") +__msg("from 4 to 1:") +__msg("loop_stack=1{{$}}") +__naked void nested_loop_backedge_to_outer_header(void) +{ + asm volatile (" \ + r6 = 0; \ +outer_%=: \ + if r6 == 3 goto exit_%=; \ + r6 += 1; \ + r7 = 0; \ +inner_%=: \ + if r7 == 2 goto outer_%=; \ + r7 += 1; \ + goto inner_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Induction variable as the src operand of the latch compare. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 2, header_count is 10") +__msg("loop header at 2, widening r1 to 0..9 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void post_cond_ivar_src(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 10; \ +1: r1 += 1; \ + if r2 != r1 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Early exit dominates the latch and gives a tighter bound. */ +SEC("socket") +__success +__log_level(2) +__msg(" backedge from 4, latch at 4, dominating exits at 2") +__msg("loop header at 2, header_count is [0..15]") +__msg("loop header at 2, widening r1 to 0..14 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void dominating_exit_bounds_count(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 14; \ +1: if r2 == r1 goto 2f; \ + r1 += 1; \ + if r1 != 63 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; From 36edb1c7738dade8ed83ec4e0aef000e538b2095 Mon Sep 17 00:00:00 2001 From: Maxim Skokov Date: Fri, 9 Oct 2026 20:47:47 +0300 Subject: [PATCH 440/660] bpf: test_run: Fix false -EBADMSG after skb leaves CHECKSUM_COMPLETE With BPF_F_TEST_SKB_CHECKSUM_COMPLETE, bpf_prog_test_run_skb() computes skb->csum over the packet from the network header to the end and marks the skb CHECKSUM_COMPLETE. After the program has run, it recomputes the checksum and returns -EBADMSG if it does not match skb->csum. bpf_skb_change_tail() resizes the skb with __skb_trim_rcsum() and __skb_grow_rcsum(), which move a CHECKSUM_COMPLETE skb to CHECKSUM_NONE and leave skb->csum as it is. The post-run check still compares against that stale value, so a correct program fails with -EBADMSG. After a trim, skb->csum still covers the removed bytes. After a grow, later writes cannot update skb->csum, even with BPF_F_RECOMPUTE_CSUM, because skb->csum is only adjusted for CHECKSUM_COMPLETE skbs. skb->csum only holds the packet checksum while ip_summed is CHECKSUM_COMPLETE. With CHECKSUM_NONE the stack does not rely on it and validates the packet in software, so there is nothing to compare. Only do the check if the skb is still CHECKSUM_COMPLETE after the run. A program that keeps the skb CHECKSUM_COMPLETE but leaves skb->csum stale still gets -EBADMSG. Dropping the checksum offload is documented behaviour of bpf_skb_change_tail() ("implicitly linearizes, unclones and drops offloads"), and the stack does the same in other places, for example skb_forward_csum(). The test harness has to accept CHECKSUM_NONE either way, whether or not the helper is later taught to keep CHECKSUM_COMPLETE. Fixes: a3cfe84cca28 ("bpf: Add CHECKSUM_COMPLETE to bpf test progs") Signed-off-by: Maxim Skokov Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009174755.22026-2-skokovmaksimevg@gmail.com --- net/bpf/test_run.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 513354e928cb..d73cd247f8cd 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1237,7 +1237,13 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, memset(__skb_push(skb, hh_len), 0, hh_len); } - if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) { + /* + * A helper such as bpf_skb_change_tail() may have downgraded the skb + * from CHECKSUM_COMPLETE. skb->csum is then unused by the stack and + * there is nothing to validate. + */ + if ((kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) && + skb->ip_summed == CHECKSUM_COMPLETE) { const int off = skb_network_offset(skb); int len = skb->len - off; __wsum csum; From 2e6be13e8ca42348ca9c68544f014f29cc677894 Mon Sep 17 00:00:00 2001 From: Maxim Skokov Date: Fri, 9 Oct 2026 20:47:48 +0300 Subject: [PATCH 441/660] selftests/bpf: Cover test_run csum validation after bpf_skb_change_tail() Add BPF_F_TEST_SKB_CHECKSUM_COMPLETE tests for tc programs that move the skb out of CHECKSUM_COMPLETE with bpf_skb_change_tail(): - change_tail_trim trims the last 4 bytes of pkt_v4. They hold the non-zero tcp.urg_ptr, so the stale skb->csum no longer matches the packet. - change_tail_grow_write appends 4 zero bytes, which on their own do not change the sum, and then writes a byte with BPF_F_RECOMPUTE_CSUM. That write can no longer update skb->csum. Both return -EBADMSG without the preceding test_run fix and pass with it. Also add store_no_recompute as a negative test. It writes a byte without BPF_F_RECOMPUTE_CSUM, so the skb stays CHECKSUM_COMPLETE with a stale skb->csum, and test_run must keep returning -EBADMSG. This checks that the validation still runs for CHECKSUM_COMPLETE skbs. The passing case, a CHECKSUM_COMPLETE skb whose checksum is updated correctly, is already covered by test_skb_pkt_end. Signed-off-by: Maxim Skokov Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009174755.22026-3-skokovmaksimevg@gmail.com --- .../bpf/prog_tests/skb_csum_complete.c | 39 ++++++++++++++++ .../selftests/bpf/progs/skb_csum_complete.c | 46 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c create mode 100644 tools/testing/selftests/bpf/progs/skb_csum_complete.c diff --git a/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c new file mode 100644 index 000000000000..919f24dbacd3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "skb_csum_complete.skel.h" + +static void run(struct bpf_program *prog, int expected_err) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + .flags = BPF_F_TEST_SKB_CHECKSUM_COMPLETE, + ); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts); + if (!ASSERT_EQ(err, expected_err, "test_run")) + return; + if (!expected_err) + ASSERT_EQ(topts.retval, 0, "retval"); +} + +void test_skb_csum_complete(void) +{ + struct skb_csum_complete *skel; + + skel = skb_csum_complete__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (test__start_subtest("change_tail_trim")) + run(skel->progs.change_tail_trim, 0); + if (test__start_subtest("change_tail_grow_write")) + run(skel->progs.change_tail_grow_write, 0); + if (test__start_subtest("store_no_recompute")) + run(skel->progs.store_no_recompute, -EBADMSG); + + skb_csum_complete__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/skb_csum_complete.c b/tools/testing/selftests/bpf/progs/skb_csum_complete.c new file mode 100644 index 000000000000..644731a12dc8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/skb_csum_complete.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +SEC("tc") +int change_tail_trim(struct __sk_buff *skb) +{ + /* + * Drops CHECKSUM_COMPLETE. skb->csum still covers the trimmed bytes, + * which hold the non-zero tcp.urg_ptr of pkt_v4, so it no longer + * matches the packet. + */ + if (bpf_skb_change_tail(skb, skb->len - 4, 0)) + return 1; + return 0; +} + +SEC("tc") +int change_tail_grow_write(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* + * Drops CHECKSUM_COMPLETE. The new bytes are zero and do not change + * the sum, but the write below can no longer update skb->csum. + */ + if (bpf_skb_change_tail(skb, skb->len + 4, 0)) + return 1; + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), + BPF_F_RECOMPUTE_CSUM)) + return 2; + return 0; +} + +SEC("tc") +int store_no_recompute(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* skb stays CHECKSUM_COMPLETE, but skb->csum is not updated. */ + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), 0)) + return 1; + return 0; +} + +char _license[] SEC("license") = "GPL"; From 6335e6719fd323c345427b7cf2d5d17c28fdbc41 Mon Sep 17 00:00:00 2001 From: Matthew Wood Date: Fri, 9 Oct 2026 10:37:59 -0700 Subject: [PATCH 442/660] bpf, arm64: Fix racy plt target update in bpf_arch_text_poke() When a long-jump trampoline is attached to or detached from a bpf prog, bpf_arch_text_poke() updates the plt target at the end of the prog by temporarily making the page writable: set_memory_rw(page); WRITE_ONCE(plt->target, plt_target); set_memory_ro(page); Since commit 1dad391daef1 ("bpf, arm64: use bpf_prog_pack for memory management"), bpf progs are allocated from the bpf prog pack, so unrelated progs and their plts commonly share a page. The sequence above is not serialized against other pokers, it runs before text_mutex is taken and callers only hold the lock of the trampoline being updated. Two CPUs attaching to or detaching from different target progs in the same page can therefore race: CPU A CPU B set_memory_rw(page) set_memory_rw(page) WRITE_ONCE(plt_B->target, ...) set_memory_ro(page) WRITE_ONCE(plt_A->target, ...) <- permission fault This was hit on an arm64 server with 64K pages while attaching fentry programs to bpf progs: Unable to handle kernel write to read-only memory at virtual address ffff80008f46d5f8 ESR = 0x000000009600004f FSC = 0x0f: level 3 permission fault pte=00c00200ea5a0783 Internal error: Oops: 000000009600004f [#1] SMP pc : bpf_arch_text_poke+0x214/0x238 lr : bpf_arch_text_poke+0x200/0x238 Call trace: bpf_arch_text_poke+0x214/0x238 (P) __bpf_trampoline_link_prog+0x1c8/0x470 bpf_trampoline_link_prog+0x64/0x90 bpf_tracing_prog_attach+0x318/0x4a8 bpf_raw_tp_link_attach+0x104/0x258 bpf_raw_tracepoint_open+0x6c/0x90 __sys_bpf+0x134c/0x3e10 Rather than serializing the permission changes, stop changing page permissions altogether and write the plt target with aarch64_insn_write_literal_u64(). It writes through the text patching fixmap under patch_lock, the same way the rest of the prog pack is written, and performs a single-copy atomic 64-bit store. The plt target is naturally aligned (see build_plt()), so CPUs concurrently executing the plt still observe either the old or the new target. This is the same helper ftrace and static calls use to update 64-bit literals that are loaded concurrently. Fixes: 1dad391daef1 ("bpf, arm64: use bpf_prog_pack for memory management") Signed-off-by: Matthew Wood Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009173802.1248441-1-thepacketgeek@gmail.com --- arch/arm64/net/bpf_jit_comp.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 08725cde0c5a..506c4201fec8 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -22,7 +22,6 @@ #include #include #include -#include #include "bpf_jit.h" @@ -3341,21 +3340,22 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, */ plt_target = (u64)&dummy_tramp; - if (plt_target) { - /* non-zero plt_target indicates we're patching a bpf prog, - * which is read only. - */ - if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), 1)) - return -EFAULT; - WRITE_ONCE(plt->target, plt_target); - set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), 1); - /* since plt target points to either the new trampoline - * or dummy_tramp, even if another CPU reads the old plt - * target value before fetching the bl instruction to plt, - * it will be brought back by dummy_tramp, so no barrier is - * required here. - */ - } + /* + * non-zero plt_target indicates we're patching a bpf prog, + * which is read only. The prog may share its page in the bpf + * prog pack with other progs, so write the aligned target + * through the text patching fixmap without flipping the + * page permissions to avoid racing with concurrent pokers. + * + * since plt target points to either the new trampoline + * or dummy_tramp, even if another CPU reads the old plt + * target value before fetching the bl instruction to plt, + * it will be brought back by dummy_tramp, so no barrier is + * required here. + */ + if (plt_target && + aarch64_insn_write_literal_u64(&plt->target, plt_target)) + return -EFAULT; /* if the old target and the new target are both long jumps, no * patching is required From 114ce944d01500ffcbf9159a72806d7f633ad9ae Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:21 +0000 Subject: [PATCH 443/660] bpf: tcp: Convert deny-list for bpf_{get,set}sockopt() to allow-list. Currently, four bpf_tcp_ops callbacks are not allowed to call bpf_setsockopt() and bpf_getsockopt(). However, the deny-list is fragile, and when a new callback is added, we might re-open a can of worms. [0][1] Let's convert it to allow-list. Note that it can be written as offsetof(,X) <= moff <= offsetof(,Y) but clang can optimise to similar code anyway. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://lore.kernel.org/r/20220929070407.965581-5-martin.lau@linux.dev/ #[0] Link: https://lore.kernel.org/bpf/20260421155804.135786-1-kafai.wan@linux.dev/ #[1] Link: https://patch.msgid.link/20261008031604.256498-2-kuniyu@google.com --- net/ipv4/bpf_tcp_ops.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index 681fed642999..1ada3b781bf1 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -210,6 +210,24 @@ const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = { .ret_type = RET_INTEGER, }; +static bool is_sockopt_supported(u32 moff) +{ + switch (moff) { + case offsetof(struct bpf_tcp_ops, active_established): + case offsetof(struct bpf_tcp_ops, passive_established): + case offsetof(struct bpf_tcp_ops, rto): + case offsetof(struct bpf_tcp_ops, rtt): + case offsetof(struct bpf_tcp_ops, set_state): + case offsetof(struct bpf_tcp_ops, retrans): + case offsetof(struct bpf_tcp_ops, connect): + case offsetof(struct bpf_tcp_ops, listen): + case offsetof(struct bpf_tcp_ops, parse_hdr): + return true; + } + + return false; +} + static const struct bpf_func_proto * get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -221,22 +239,13 @@ get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_storage_delete: return &bpf_sk_storage_delete_proto; case BPF_FUNC_setsockopt: - /* The sk may be an unlocked listener (synack path) or NULL - * fullsock; disable for members that can run unlocked. - */ - if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || - moff == offsetof(struct bpf_tcp_ops, timeout_init) || - moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) || - moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) - return NULL; - return &bpf_sk_setsockopt_proto; + if (is_sockopt_supported(moff)) + return &bpf_sk_setsockopt_proto; + return NULL; case BPF_FUNC_getsockopt: - if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) || - moff == offsetof(struct bpf_tcp_ops, timeout_init) || - moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) || - moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) - return NULL; - return &bpf_sk_getsockopt_proto; + if (is_sockopt_supported(moff)) + return &bpf_sk_getsockopt_proto; + return NULL; case BPF_FUNC_get_retval: if (moff == offsetof(struct bpf_tcp_ops, timeout_init) || moff == offsetof(struct bpf_tcp_ops, rwnd_init)) From 4e2385abecb03998922afbfb91caff273fe6cbee Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:22 +0000 Subject: [PATCH 444/660] bpf: tcp: Add a new per-socket flag and kfunc for bpf_tcp_ops. The legacy SOCK_OPS guards some hooks with a per-socket flag, tp->bpf_sock_ops_cb_flags. In contrast, bpf_tcp_ops was initially designed without per-socket flags so that users can simply define only the callbacks they need. However, it turned out that even attaching a bpf_tcp_ops with a NULL callback incurs measurable overhead in the fast path. [0] We could reuse tp->bpf_sock_ops_cb_flags, but it is u8 and has only 1 bit left for a new opt-in callback, while not all of the 7 existing opt-in hooks are in the fast path and really need a flag guard. Moreover, tp->bpf_sock_ops_cb_flags has two bpf helper functions to update it, both of which require sock_owned_by_me(sk): * bpf_sock_ops_cb_flags_set() * bpf_setsockopt(TCP_BPF_SOCK_OPS_CB_FLAGS) Both helpers only overwrite the field, which leads to reading and modifying the flags in the BPF prog and then writing them back via the helper. This prevents use from the fast path (tc or cgroup_skb) hooks where bh_lock_sock() alone cannot prevent races with process context. Let's add a new u32 field, tp->bpf_tcp_ops_flags, at the end of the tcp_sock_read_txrx cacheline group, along with a new kfunc, bpf_tcp_ops_set_flags(). bpf_tcp_ops_set_flags() takes bitmasks of flags to enable and disable and updates tp->bpf_tcp_ops_flags atomically via try_cmpxchg() without relying on lock_sock(). The kfunc is exposed to bpf_tcp_ops, BPF_PROG_TYPE_CGROUP_SOCKOPT, and BPF_PROG_TYPE_CGROUP_SKB. The first argument is struct tcp_sock * so that bpf_tcp_sock() etc is required for cgroup hooks, but not for bpf_tcp_ops where struct sock * is promoted to struct tcp_sock * automatically. The fast-path callbacks will be guarded by the new flags in a later patch after updating the existing selftest to avoid breaking bisection. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://lore.kernel.org/netdev/CAMB2axMwBuz3X4Uwn5uzZUqm91EbiMnbQX832wUFOQ44fOwDnQ@mail.gmail.com/ #[0] Link: https://patch.msgid.link/20261008031604.256498-3-kuniyu@google.com --- .../networking/net_cachelines/tcp_sock.rst | 1 + include/linux/tcp.h | 7 +++ include/net/tcp.h | 11 +++- include/uapi/linux/bpf.h | 23 +++++++ net/ipv4/bpf_tcp_ops.c | 61 ++++++++++++++++++- net/ipv4/tcp.c | 3 + tools/include/uapi/linux/bpf.h | 23 +++++++ 7 files changed, 127 insertions(+), 2 deletions(-) diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst index 0f6088c4ab8b..420fc6278148 100644 --- a/Documentation/networking/net_cachelines/tcp_sock.rst +++ b/Documentation/networking/net_cachelines/tcp_sock.rst @@ -151,6 +151,7 @@ u32 urg_seq unsigned_int keepalive_time unsigned_int keepalive_intvl int linger2 +u32 bpf_tcp_ops_flags read_mostly read_mostly bpf_tcp_ops_hdr_opt_len,bpf_skops_write_hdr_opt(tx);bpf_tcp_ops_parse_hdr,tcp_bpf_rtt(rx); u8 bpf_sock_ops_cb_flags u8:1 bpf_chg_cc_inprogress u16 timeout_rehash diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 6a8c77719322..24bb751cc65a 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -233,6 +233,13 @@ struct tcp_sock { is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ +#ifdef CONFIG_BPF + u32 bpf_tcp_ops_flags; +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) \ + (READ_ONCE((TP)->bpf_tcp_ops_flags) & BPF_TCP_OPS_FLAG_ ## ARG) +#else +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) (0) +#endif __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 4ecabf4989de..55ad9db99596 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -2934,6 +2934,7 @@ static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; + WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0); } #else @@ -2989,7 +2990,7 @@ struct bpf_tcp_ops { /* Called when the retransmission timer fires. */ void (*rto)(struct sock *sk); - /* Called on every RTT sample. + /* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled. * @mrtt: the measured RTT, in microseconds. * @srtt: the updated smoothed RTT. */ @@ -3016,6 +3017,10 @@ struct bpf_tcp_ops { * Parse the TCP header options of an incoming skb received on an * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() * to access the options. + * + * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if + * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown + * option is received. */ void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); @@ -3023,6 +3028,8 @@ struct bpf_tcp_ops { * Reserve space in the outgoing TCP header for options to be written * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * * @skb: outgoing packet. NULL when called from tcp_current_mss() * (MSS sizing). * @req: request_sock on the synack path; NULL otherwise. @@ -3041,6 +3048,8 @@ struct bpf_tcp_ops { * Use bpf_store_hdr_opt() to write; it appends within the reserved window * shared with legacy SOCKOPS. * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * * @skb: outgoing packet. * @req: request_sock on the synack path; NULL otherwise. * @syn_skb: incoming SYN on the synack path; NULL otherwise. diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index e0ed44b1bbcb..6963c146311e 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -7340,6 +7340,29 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + BPF_TCP_OPS_FLAG_ALL = (1 << 4) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index 1ada3b781bf1..aaf96304c69f 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -328,8 +328,67 @@ static struct bpf_struct_ops bpf_tcp_ops = { .owner = THIS_MODULE, }; +__bpf_kfunc_start_defs(); + +__bpf_kfunc int bpf_tcp_ops_set_flags(struct tcp_sock *tp, u32 enable, u32 disable) +{ + u32 old, new; + + if ((enable & disable) || (enable | disable) & ~BPF_TCP_OPS_FLAG_ALL) + return -EINVAL; + + old = READ_ONCE(tp->bpf_tcp_ops_flags); + + do { + new = (old | enable) & ~disable; + if (new == old) + break; + } while (!try_cmpxchg(&tp->bpf_tcp_ops_flags, &old, new)); + + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(bpf_tcp_ops_set_flags_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_flags) +BTF_KFUNCS_END(bpf_tcp_ops_set_flags_kfunc_set) + +static int bpf_tcp_ops_set_flags_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + if (!btf_id_set8_contains(&bpf_tcp_ops_set_flags_kfunc_set, kfunc_id)) + return 0; + + if (prog->type == BPF_PROG_TYPE_STRUCT_OPS && + prog->aux->st_ops == &bpf_tcp_ops) + return 0; + + if (prog->type == BPF_PROG_TYPE_CGROUP_SOCKOPT || + prog->type == BPF_PROG_TYPE_CGROUP_SKB) + return 0; + + return -EACCES; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_flags_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_flags_kfunc_set, + .filter = bpf_tcp_ops_set_flags_kfunc_filter, +}; + static int __init __bpf_tcp_ops_init(void) { - return register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_flags_kfunc_id_set); + /* BPF_PROG_TYPE_CGROUP_{SOCKOPT,SKB} share BTF_KFUNC_HOOK_CGROUP. */ + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCKOPT, + &bpf_tcp_ops_set_flags_kfunc_id_set); + ret = ret ?: register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); + + return ret; } + late_initcall(__bpf_tcp_ops_init); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 650a2e89950a..fa69961c47d3 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -5217,6 +5217,9 @@ static void __init tcp_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio); +#ifdef CONFIG_BPF + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, bpf_tcp_ops_flags); +#endif /* RX read-mostly hotpath cache lines */ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e0ed44b1bbcb..6963c146311e 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -7340,6 +7340,29 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + BPF_TCP_OPS_FLAG_ALL = (1 << 4) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling From ca577d08f88b410bc1b87d93f70bf68c03083b96 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:23 +0000 Subject: [PATCH 445/660] selftest: bpf: Use bpf_tcp_ops_set_flags() in bpf_tcp_ops_hdr.c. The next patch guards bpf_tcp_ops callbacks in the fast path with per-socket flags. Let's use bpf_tcp_ops_set_flags() to enable flags for parse_hdr, write_hdr_opt, and hdr_opt_len. Note that BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL and BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN are used on the server and client, respectively, for better coverage. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-4-kuniyu@google.com --- .../selftests/bpf/progs/bpf_tcp_ops_hdr.c | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c index f3e3dff13784..706e94d794e4 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c @@ -18,6 +18,24 @@ int found_cnt; __u8 found_d0; __u8 found_d1; +SEC("struct_ops") +void BPF_PROG(test_listen, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + +SEC("struct_ops") +void BPF_PROG(test_connect, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + SEC("struct_ops") void BPF_PROG(test_hdr_opt_len, struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct sk_buff *syn_skb, @@ -78,6 +96,8 @@ void BPF_PROG(test_parse_hdr, struct sock *sk, struct sk_buff *skb) SEC(".struct_ops.link") struct bpf_tcp_ops test_hdr_ops = { + .listen = (void *)test_listen, + .connect = (void *)test_connect, .hdr_opt_len = (void *)test_hdr_opt_len, .write_hdr_opt = (void *)test_write_hdr_opt, .parse_hdr = (void *)test_parse_hdr, From 59c25bb6759ae445d2bd431e7166cebec1afa9a2 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:24 +0000 Subject: [PATCH 446/660] bpf: tcp: Guard fast-path bpf_tcp_ops_call() under per-socket flag. bpf_tcp_ops.{parse_hdr,hdr_opt_len} are called for every incoming / outgoing skb. bpf_tcp_ops.rtt is called once per RTT, which is every incoming skb in ping-pong workloads like tcp_rr. Even attaching NULL callbacks in the fast path hurts performance. Let's guard them (and write_hdr_opt) with the new per-socket flags. __bpf_tcp_ops_call() and bpf_tcp_ops_call_flag() are added to check cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) first and avoid accessing tp->bpf_tcp_ops_flags when no bpf_tcp_ops is attached. Note that bpf_tcp_ops_hdr_opt_len() has 3 callers and previously checked the static key both in tcp_established_options() and inside bpf_tcp_ops_call(). Now the static key is checked once at the beginning of bpf_tcp_ops_hdr_opt_len(), and the redundant check in tcp_established_options() is removed. tcp_bpf_hdr_opt_len() must use __always_inline, not just inline, otherwise Clang inlines bpf_tcp_ops_hdr_opt_len() to tcp_bpf_hdr_opt_len() instead. $ llvm-objdump -S -D --disassemble=tcp_established_options vmlinux ... ; if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, ffffffff8251270e: 41 f6 87 20 0c 00 00 40 testb $0x40, 0xc20(%r15) ffffffff82512716: 75 47 jne 0xffffffff8251275f ; asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]") ffffffff82512718: 0f 1f 44 00 00 nopl (%rax,%rax) ; return size; ffffffff8251271d: 44 89 e0 movl %r12d, %eax ffffffff82512720: 5b popq %rbx ffffffff82512721: 41 5c popq %r12 ffffffff82512723: 41 5e popq %r14 ffffffff82512725: 41 5f popq %r15 ffffffff82512727: 5d popq %rbp ffffffff82512728: 2e e9 42 03 43 00 jmp 0xffffffff82942a70 <__x86_return_thunk> bpf_skops_write_hdr_opt() is still not free, but it can be optimised later if needed. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-5-kuniyu@google.com --- include/net/tcp.h | 46 +++++++++++++++++++++++------------- net/ipv4/tcp_input.c | 12 +++++++++- net/ipv4/tcp_output.c | 55 ++++++++++++++++++++++++++----------------- 3 files changed, 75 insertions(+), 38 deletions(-) diff --git a/include/net/tcp.h b/include/net/tcp.h index 55ad9db99596..b7c0f1a8797a 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -3064,22 +3064,33 @@ struct bpf_tcp_ops { u32 opt_off); }; +#define __bpf_tcp_ops_call(op, sk, ...) \ +do { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct cgroup *cgrp; \ + \ + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ + rcu_read_lock_dont_migrate(); \ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + tcp_ops->op(sk, ##__VA_ARGS__); \ + } \ + rcu_read_unlock_migrate(); \ +} while (0) + #define bpf_tcp_ops_call(op, sk, ...) \ do { \ - if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ - const struct bpf_prog_array_item *item; \ - const struct bpf_tcp_ops *tcp_ops; \ - struct cgroup *cgrp; \ - \ - cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ - rcu_read_lock_dont_migrate(); \ - bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ - CGROUP_TCP_SOCK_OPS) { \ - if (tcp_ops->op) \ - tcp_ops->op(sk, ##__VA_ARGS__); \ - } \ - rcu_read_unlock_migrate(); \ - } \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \ + BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ } while (0) #define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ @@ -3115,7 +3126,9 @@ do { \ }) #else -#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define __bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0) #define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) #endif @@ -3150,7 +3163,8 @@ static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); - bpf_tcp_ops_call(rtt, sk, mrtt, srtt); + + bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); } #if IS_ENABLED(CONFIG_SMC) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 209db8effcc4..4478d3f3d4b0 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -210,6 +210,11 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb) { + const struct tcp_sock *tp; + + if (!cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + return; + switch (sk->sk_state) { case TCP_SYN_RECV: case TCP_SYN_SENT: @@ -217,7 +222,12 @@ static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb) return; } - bpf_tcp_ops_call(parse_hdr, sk, skb); + tp = tcp_sk(sk); + + if ((tp->rx_opt.saw_unknown && + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_UNKNOWN)) || + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_ALL)) + __bpf_tcp_ops_call(parse_hdr, sk, skb); } static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb, diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 3ac465513bf5..313dfe70a386 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -581,8 +581,8 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, * writer's bytes). The writer finds the append point by scanning from * first_opt_off + nr_written to the first NOP. */ - bpf_tcp_ops_call(write_hdr_opt, sk, skb, req, syn_skb, synack_type, - first_opt_off + nr_written); + bpf_tcp_ops_call_flag(write_hdr_opt, WRITE_HDR_OPT, sk, skb, req, + syn_skb, synack_type, first_opt_off + nr_written); } #else static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, @@ -613,11 +613,12 @@ static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, { unsigned int remaining_out = remaining, reserved; - if (!remaining) - return 0; + if (!BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), WRITE_HDR_OPT) || + !remaining) + return remaining; /* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */ - bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out); + __bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out); reserved = remaining - remaining_out; if (!reserved) @@ -630,6 +631,21 @@ static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, return remaining - reserved; } +static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + + return remaining; +} + static __be32 *process_tcp_ao_options(struct tcp_sock *tp, const struct tcp_request_sock *tcprsk, struct tcp_out_options *opts, @@ -1089,8 +1105,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, remaining); - remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1179,8 +1195,8 @@ static unsigned int tcp_synack_options(const struct sock *sk, remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, synack_type, opts, remaining); - remaining = bpf_tcp_ops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, - synack_type, opts, remaining); + remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, + synack_type, opts, remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1193,8 +1209,9 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb struct tcp_key *key) { struct tcp_sock *tp = tcp_sk(sk); - unsigned int size = 0; unsigned int eff_sacks; + unsigned int remaining; + unsigned int size = 0; opts->options = 0; opts->bpf_opt_len = 0; @@ -1224,10 +1241,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb * left. */ if (sk_is_mptcp(sk)) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; bool has_ts = opts->options & OPTION_TS; int opt_size; + remaining = MAX_TCP_OPTION_SPACE - size; opts->mptcp.drop_ts = 0; opt_size = mptcp_established_options(sk, skb, remaining, has_ts, @@ -1244,7 +1261,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; if (unlikely(eff_sacks)) { - const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = MAX_TCP_OPTION_SPACE - size; + if (likely(remaining >= TCPOLEN_SACK_BASE_ALIGNED + TCPOLEN_SACK_PERBLOCK)) { opts->num_sack_blocks = @@ -1277,22 +1295,17 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; - + remaining = MAX_TCP_OPTION_SPACE - size; remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, remaining); size = MAX_TCP_OPTION_SPACE - remaining; } - if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + MAX_TCP_OPTION_SPACE - size); - remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); - - size = MAX_TCP_OPTION_SPACE - remaining; - } + size = MAX_TCP_OPTION_SPACE - remaining; return size; } From c071408db308a8f91d7be71b45bbf2da8c5e15f4 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:25 +0000 Subject: [PATCH 447/660] bpf: tcp: Check BPF_SOCK_OPS_TEST_FLAG() after cgroup_bpf_enabled(CGROUP_SOCK_OPS). BPF_CGROUP_RUN_PROG_XXX() macros guard __cgroup_bpf_run_filter_XXX() with cgroup_bpf_enabled(). However, even when no SOCK_OPS prog is attached, callers still initialise struct bpf_sock_ops_kern (memset(), etc.) or evaluate BPF_SOCK_OPS_TEST_FLAG(), which loads tp->bpf_sock_ops_cb_flags from a cold cacheline near the end of struct tcp_sock. Similar to bpf_tcp_ops, let's check cgroup_bpf_enabled() before BPF_SOCK_OPS_TEST_FLAG() and struct bpf_sock_ops_kern setup, and rename BPF_CGROUP_RUN_PROG_SOCK_OPS{,_SK}() with __ prefix. Since all direct callers of tcp_call_bpf() pass 0 and NULL for nargs and args, they can be folded into the new tcp_call_bpf() macro. All callers of tcp_call_bpf_{2,3}arg() check BPF_SOCK_OPS_TEST_FLAG() and do not need the return value. These are replaced with the new tcp_call_bpf_flag() macro. Now, bpf_skops_hdr_opt_len() and bpf_tcp_ops_hdr_opt_len() are just NOPs without bpf prog attached. $ llvm-objdump -S -D --disassemble=tcp_established_options vmlinux ... ; asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]") ; size = MAX_TCP_OPTION_SPACE - remaining; ffffffff825127ea: b9 28 00 00 00 movl $0x28, %ecx ffffffff825127ef: 29 c1 subl %eax, %ecx ; return size; ffffffff825127f1: 89 c8 movl %ecx, %eax ... ffffffff825127fb: 2e e9 6f 02 43 00 jmp 0xffffffff82942a70 <__x86_return_thunk> Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-6-kuniyu@google.com --- include/linux/bpf-cgroup.h | 34 +++++++++-------------- include/net/tcp.h | 56 ++++++++++++++++---------------------- net/ipv4/af_inet.c | 2 +- net/ipv4/tcp.c | 3 +- net/ipv4/tcp_input.c | 21 ++++++++------ net/ipv4/tcp_nv.c | 2 +- net/ipv4/tcp_output.c | 36 ++++++++++-------------- net/ipv4/tcp_timer.c | 6 ++-- 8 files changed, 69 insertions(+), 91 deletions(-) diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 8a75a6cd7309..02ef7f899b3d 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -345,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, * calling bpf_setsockopt on listener-sk will not make sense anyway, * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. */ -#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - __ret; \ -}) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ + __cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \ + CGROUP_SOCK_OPS) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ - typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ - if (__sk && sk_fullsock(__sk)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - } \ - __ret; \ +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ +({ \ + int __ret = 0; \ + typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ + if (__sk && sk_fullsock(__sk)) \ + __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ + sock_ops, \ + CGROUP_SOCK_OPS); \ + __ret; \ }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \ @@ -529,7 +521,7 @@ static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; }) #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ diff --git a/include/net/tcp.h b/include/net/tcp.h index b7c0f1a8797a..85b4bfe963d3 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -2891,7 +2891,7 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, * program loaded). */ #ifdef CONFIG_BPF -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { struct bpf_sock_ops_kern sock_ops; int ret; @@ -2908,7 +2908,7 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) if (nargs > 0) memcpy(sock_ops.args, args, nargs * sizeof(*args)); - ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); if (ret == 0) ret = sock_ops.reply; else @@ -2916,20 +2916,23 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) return ret; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - u32 args[2] = {arg1, arg2}; +#define tcp_call_bpf(sk, op) \ +({ \ + int __ret = 0; \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \ + __ret = __tcp_call_bpf(sk, op, 0, NULL); \ + } \ + __ret; \ +}) - return tcp_call_bpf(sk, op, 2, args); -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - u32 args[3] = {arg1, arg2, arg3}; - - return tcp_call_bpf(sk, op, 3, args); -} +#define tcp_call_bpf_flag(sk, op, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \ + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \ + u32 __args[] = { __VA_ARGS__ }; \ + __tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \ + } \ +} while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { @@ -2938,21 +2941,12 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) } #else -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int tcp_call_bpf(struct sock *sk, int op) { return -EPERM; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - return -EPERM; -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - return -EPERM; -} +#define tcp_call_bpf_flag(sk, op, ...) do { } while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { @@ -3136,7 +3130,7 @@ static inline u32 tcp_timeout_init(struct sock *sk) { int timeout; - timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); + timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); if (timeout <= 0) timeout = TCP_TIMEOUT_INIT; @@ -3147,7 +3141,7 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) { int rwnd; - rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); + rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); if (rwnd < 0) rwnd = 0; @@ -3156,14 +3150,12 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) { - return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); + return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); } static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); - + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); } diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index cdcfc7d3c6d2..035c83338e0f 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -226,7 +226,7 @@ int __inet_listen_sk(struct sock *sk, int backlog) if (err) return err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB); bpf_tcp_ops_call(listen, sk); } return 0; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index fa69961c47d3..5d9d3bcde8f7 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2994,8 +2994,7 @@ void tcp_set_state(struct sock *sk, int state) */ BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); bpf_tcp_ops_call(set_state, sk, state); switch (state) { diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 4478d3f3d4b0..f374257013b1 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -146,14 +146,16 @@ EXPORT_SYMBOL_GPL(clean_acked_data_flush); #ifdef CONFIG_CGROUP_BPF static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) { - bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown && - BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG); - bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG); struct bpf_sock_ops_kern sock_ops; + const struct tcp_sock *tp; - if (likely(!unknown_opt && !parse_all_opt)) + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + + tp = tcp_sk(sk); + if (!(tp->rx_opt.saw_unknown && + BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG)) && + !BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG)) return; /* The skb will be handled in the @@ -176,7 +178,7 @@ static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) sock_ops.sk = sk; bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } static void bpf_skops_established(struct sock *sk, int bpf_op, @@ -184,6 +186,9 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, { struct bpf_sock_ops_kern sock_ops; + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + sock_owned_by_me(sk); memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); @@ -195,7 +200,7 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, if (skb) bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } #else static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c index f345897a68df..7b0dae23d9aa 100644 --- a/net/ipv4/tcp_nv.c +++ b/net/ipv4/tcp_nv.c @@ -146,7 +146,7 @@ static void tcpnv_init(struct sock *sk) * within a datacenter, where we have reasonable estimates of * RTTs */ - base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT, 0, NULL); + base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT); if (base_rtt > 0) { ca->nv_base_rtt = base_rtt; ca->nv_lower_bound_rtt = (base_rtt * 205) >> 8; /* 80% */ diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 313dfe70a386..d4ee359d36d0 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -476,8 +476,8 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, struct bpf_sock_ops_kern sock_ops; int err; - if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) || + if (!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG) || !remaining) return remaining; @@ -518,7 +518,7 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, if (skb) bpf_skops_init_skb(&sock_ops, skb, 0); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); if (err || sock_ops.remaining_opt_len == remaining) return remaining; @@ -543,7 +543,8 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, first_opt_off = tcp_hdrlen(skb) - max_opt_len; - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) { struct bpf_sock_ops_kern sock_ops; int err; @@ -567,7 +568,7 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, sock_ops.remaining_opt_len = max_opt_len; bpf_skops_init_skb(&sock_ops, skb, first_opt_off); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); if (!err) nr_written = max_opt_len - sock_ops.remaining_opt_len; } @@ -638,6 +639,11 @@ static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff * struct tcp_out_options *opts, u32 remaining) { + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + remaining = bpf_skops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb, synack_type, opts, @@ -1103,8 +1109,6 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, remaining); @@ -1193,8 +1197,6 @@ static unsigned int tcp_synack_options(const struct sock *sk, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, - synack_type, opts, remaining); remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, synack_type, opts, remaining); @@ -1293,15 +1295,6 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb } } - if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) { - remaining = MAX_TCP_OPTION_SPACE - size; - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); - - size = MAX_TCP_OPTION_SPACE - remaining; - } - remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, MAX_TCP_OPTION_SPACE - size); @@ -3738,9 +3731,8 @@ start: err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); } - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB, - TCP_SKB_CB(skb)->seq, segs, err); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RETRANS_CB, + TCP_SKB_CB(skb)->seq, segs, err); bpf_tcp_ops_call(retrans, sk, skb, err); if (unlikely(err) && err != -EBUSY) @@ -4368,7 +4360,7 @@ int tcp_connect(struct sock *sk) struct sk_buff *buff; int err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB); bpf_tcp_ops_call(connect, sk); #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO) diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 3d49adc51766..00debcba122b 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -286,10 +286,8 @@ static int tcp_write_timeout(struct sock *sk) tcp_fastopen_active_detect_blackhole(sk, expired); mptcp_active_detect_blackhole(sk, expired); - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RTO_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB, - icsk->icsk_retransmits, - icsk->icsk_rto, (int)expired); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTO_CB, + icsk->icsk_retransmits, icsk->icsk_rto, (int)expired); bpf_tcp_ops_call(rto, sk); if (expired) { From 32d17fe8a9a5d83b8b4fe1fb453193221aec4a9b Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:26 +0000 Subject: [PATCH 448/660] bpf: tcp: Introduce bpf_tcp_ops.{enqueue,dequeue}_rcvq(). Waking up a thread per packet is expensive when an application processes variable-length frames (e.g., RPC) that span multiple packets. SO_RCVLOWAT can defer wakeups, but because the frame size is encoded in a fixed-size descriptor at the start of each frame, the application has to: 1. wake up and recv() the descriptor, 2. raise SO_RCVLOWAT to the payload size via setsockopt(), 3. wake up and recv() the payload, and 4. reset SO_RCVLOWAT back to the descriptor size via setsockopt() for the next frame. This requires an extra wakeup and two setsockopt() syscalls for every single RPC frame. With SOCKMAP, we can parse skb and suppress wakeups in kernel, but SOCKMAP adds overhead and also kills zerocopy. Let's add lighter-weight opt-in callbacks to bpf_tcp_ops to replace that. .enqueue_rcvq(): invoked when TCP stack enqueues skb to sk->sk_receive_queue .dequeue_rcvq(): invoked in tcp_cleanup_rbuf() after data is dequeued from sk->sk_receive_queue Those callbacks can be enabled on a per-socket basis by bpf_tcp_ops_set_flags(): bpf_tcp_ops_set_flags((struct tcp_sock *)sk, BPF_TCP_OPS_FLAG_RCVQ, 0); Later, we will add a new kfunc to adjust sk->sk_rcvlowat from these callbacks. This will allow the bpf_tcp_ops prog to parse each skb and dynamically adjust sk->sk_rcvlowat to suppress unnecessary EPOLLIN wakeups until sufficient data is available in the receive queue. The placement of bpf_tcp_ops_call() in tcp_ofo_queue() and tcp_fastopen_add_skb() is chosen to provide the same snapshot as tcp_queue_rcv(). For example, if bpf_tcp_ops_call() were called before updating TCP_SKB_CB(skb)->seq in tcp_fastopen_add_skb(), BPF prog would need an extra branch for the unlikely TFO case to strip SYN. In addition, the TCP stack can queue overlapping skbs into recvq. Once rcv_nxt is updated with a new skb, BPF prog can no longer infer the previous rcv_nxt from skb->len. Lastly, dequeue_rcvq() is placed in tcp_cleanup_rbuf() rather than __tcp_cleanup_rbuf() so that it is not called for sockets in SOCKMAP, where calling sk->sk_data_ready() from the new kfunc would otherwise trigger infinite recursion. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-7-kuniyu@google.com --- Documentation/networking/net_cachelines/tcp_sock.rst | 2 +- include/net/tcp.h | 12 ++++++++++++ include/uapi/linux/bpf.h | 4 +++- net/ipv4/bpf_tcp_ops.c | 10 ++++++++++ net/ipv4/tcp.c | 2 ++ net/ipv4/tcp_fastopen.c | 2 ++ net/ipv4/tcp_input.c | 4 ++++ tools/include/uapi/linux/bpf.h | 4 +++- 8 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst index 420fc6278148..e4776905e882 100644 --- a/Documentation/networking/net_cachelines/tcp_sock.rst +++ b/Documentation/networking/net_cachelines/tcp_sock.rst @@ -151,7 +151,7 @@ u32 urg_seq unsigned_int keepalive_time unsigned_int keepalive_intvl int linger2 -u32 bpf_tcp_ops_flags read_mostly read_mostly bpf_tcp_ops_hdr_opt_len,bpf_skops_write_hdr_opt(tx);bpf_tcp_ops_parse_hdr,tcp_bpf_rtt(rx); +u32 bpf_tcp_ops_flags read_mostly read_mostly bpf_tcp_ops_hdr_opt_len,bpf_skops_write_hdr_opt(tx);bpf_tcp_ops_parse_hdr,tcp_bpf_rtt,tcp_cleanup_rbuf,tcp_queue_rcv,tcp_ofo_queue,tcp_fastopen_add_skb(rx); u8 bpf_sock_ops_cb_flags u8:1 bpf_chg_cc_inprogress u16 timeout_rehash diff --git a/include/net/tcp.h b/include/net/tcp.h index 85b4bfe963d3..d95cbe4da96e 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -3056,6 +3056,18 @@ struct bpf_tcp_ops { struct request_sock *req, struct sk_buff *syn_skb, enum tcp_synack_type synack_type, u32 opt_off); + + /* + * Called when an incoming skb is enqueued to sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb); + + /* + * Called after data is dequeued from sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*dequeue_rcvq)(struct sock *sk); }; #define __bpf_tcp_ops_call(op, sk, ...) \ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 6963c146311e..6f70db7515dd 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -7360,7 +7360,9 @@ enum { BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), /* .hdr_opt_len() and .write_hdr_opt() */ BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), - BPF_TCP_OPS_FLAG_ALL = (1 << 4) - 1, + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, }; /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index aaf96304c69f..9a6e1c47d7c0 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -76,6 +76,14 @@ static void write_hdr_opt_stub(struct sock *sk, struct sk_buff *skb, { } +static void enqueue_rcvq_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void dequeue_rcvq_stub(struct sock *sk) +{ +} + static struct bpf_tcp_ops __bpf_tcp_ops = { .timeout_init = timeout_init_stub, .rwnd_init = rwnd_init_stub, @@ -90,6 +98,8 @@ static struct bpf_tcp_ops __bpf_tcp_ops = { .parse_hdr = parse_hdr_stub, .hdr_opt_len = hdr_opt_len_stub, .write_hdr_opt = write_hdr_opt_stub, + .enqueue_rcvq = enqueue_rcvq_stub, + .dequeue_rcvq = dequeue_rcvq_stub, }; BPF_CALL_4(bpf_tcp_ops_store_hdr_opt, void *, ctx, const void *, from, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 5d9d3bcde8f7..5d907a0a0461 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1609,6 +1609,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); __tcp_cleanup_rbuf(sk, copied); + + bpf_tcp_ops_call_flag(dequeue_rcvq, RCVQ, sk); } static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 471c78be5513..6dfe40322fb5 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -281,6 +281,8 @@ void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->seq++; TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN; + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; tcp_add_receive_queue(sk, skb); tp->syn_data_acked = 1; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index f374257013b1..46f5c4fafa8d 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5365,6 +5365,8 @@ static void tcp_ofo_queue(struct sock *sk) continue; } + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tail = skb_peek_tail(&sk->sk_receive_queue); eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen); tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); @@ -5568,6 +5570,8 @@ static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int eaten; struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue); + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + eaten = (tail && tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6963c146311e..6f70db7515dd 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -7360,7 +7360,9 @@ enum { BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), /* .hdr_opt_len() and .write_hdr_opt() */ BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), - BPF_TCP_OPS_FLAG_ALL = (1 << 4) - 1, + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, }; /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect From dcddae18f5128266b7ae2c41120a4cf560548286 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:27 +0000 Subject: [PATCH 449/660] tcp: Split out __tcp_set_rcvlowat(). We will add a kfunc for bpf_tcp_ops.{enqueue,dequeue}_rcvq() to adjust sk->sk_rcvlowat. These hooks are triggered * when the TCP stack enqueues an skb to sk->sk_receive_queue * after data is dequeued from sk->sk_receive_queue In the enqueue path, tcp_data_ready() is always called after the hooks in tcp_queue_rcv() and tcp_ofo_queue(). If tcp_set_rcvlowat() were used as is, tcp_data_ready() could be called twice for the same skb, which is redundant and also confusing. Let's split out __tcp_set_rcvlowat() and add a flag to control wakeup behaviour. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-8-kuniyu@google.com --- include/net/tcp.h | 1 + net/ipv4/tcp.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/net/tcp.h b/include/net/tcp.h index d95cbe4da96e..f7624ab9b420 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -512,6 +512,7 @@ void tcp_set_keepalive(struct sock *sk, int val); void tcp_syn_ack_timeout(const struct request_sock *req); int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags); +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup); int tcp_set_rcvlowat(struct sock *sk, int val); void tcp_set_rcvbuf(struct sock *sk, int val); int tcp_set_window_clamp(struct sock *sk, int val); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 5d907a0a0461..c3c756a1ba8c 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1827,8 +1827,7 @@ int tcp_peek_len(struct socket *sock) return tcp_inq(sock->sk); } -/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ -int tcp_set_rcvlowat(struct sock *sk, int val) +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup) { struct tcp_sock *tp = tcp_sk(sk); int space, cap; @@ -1841,7 +1840,8 @@ int tcp_set_rcvlowat(struct sock *sk, int val) WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); /* Check if we need to signal EPOLLIN right now */ - tcp_data_ready(sk); + if (wakeup) + tcp_data_ready(sk); if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) return 0; @@ -1856,6 +1856,12 @@ int tcp_set_rcvlowat(struct sock *sk, int val) return 0; } +/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ +int tcp_set_rcvlowat(struct sock *sk, int val) +{ + return __tcp_set_rcvlowat(sk, val, true); +} + void tcp_set_rcvbuf(struct sock *sk, int val) { tcp_set_window_clamp(sk, tcp_win_from_space(sk, val)); From d65dc83603eb6b3c639c66baf4b96593d9814d3d Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:28 +0000 Subject: [PATCH 450/660] bpf: mptcp: Don't support BPF_TCP_OPS_FLAG_RCVQ. The next patch exposes a new kfunc calling __tcp_set_rcvlowat() to bpf_tcp_ops. MPTCP has its own sock->ops->set_rcvlowat() / mptcp_set_rcvlowat(), so we should not allow calling __tcp_set_rcvlowat() on MPTCP subflows. Let's disable BPF_TCP_OPS_FLAG_RCVQ for MPTCP for now. If needed in the future, bpf_tcp_ops_set_rcvlowat() could be extended to properly support MPTCP. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-9-kuniyu@google.com --- net/ipv4/bpf_tcp_ops.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index 9a6e1c47d7c0..8182037c4269 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -347,6 +347,9 @@ __bpf_kfunc int bpf_tcp_ops_set_flags(struct tcp_sock *tp, u32 enable, u32 disab if ((enable & disable) || (enable | disable) & ~BPF_TCP_OPS_FLAG_ALL) return -EINVAL; + if (sk_is_mptcp((struct sock *)tp) && (enable & BPF_TCP_OPS_FLAG_RCVQ)) + return -EOPNOTSUPP; + old = READ_ONCE(tp->bpf_tcp_ops_flags); do { From 5f549c1e49f3130bcebcf1d6b17d7802215f2025 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:29 +0000 Subject: [PATCH 451/660] bpf: tcp: Add kfunc to adjust sk->sk_rcvlowat. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bpf_tcp_ops.{enqueue,dequeue}_rcvq() were added to parse skb and adjust sk->sk_rcvlowat dynamically to suppress unnecessary wakeups. Let's add a new kfunc to set sk->sk_rcvlowat. Negative values are clamped to INT_MAX, consistent with SO_RCVLOWAT. For enqueue_rcvq(), wakeup is set to false because: * tcp_data_ready() is always called after the hooks in tcp_queue_rcv() and tcp_ofo_queue(). * when tcp_fastopen_add_skb() is called for TFO SYN, the socket is not yet accept()ed, and when called for TFO SYN+ACK, the socket is woken up by sk->sk_state_change() anyway. For dequeue_rcvq(), wakeup is set to true because tcp_data_ready() is not called in that path. An alternative would be to support bpf_setsockopt() for these hooks. However, that approach involves excessive conditionals and an unnecessary memcpy(), costs we do not want to pay for every skb in the TCP fast path. Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Tested-by: Clément Léger Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-10-kuniyu@google.com --- net/ipv4/bpf_tcp_ops.c | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c index 8182037c4269..2ba73dd6c52c 100644 --- a/net/ipv4/bpf_tcp_ops.c +++ b/net/ipv4/bpf_tcp_ops.c @@ -361,12 +361,31 @@ __bpf_kfunc int bpf_tcp_ops_set_flags(struct tcp_sock *tp, u32 enable, u32 disab return 0; } +__bpf_kfunc int bpf_tcp_ops_set_rcvlowat(struct sock *sk, int rcvlowat, + const struct bpf_prog_aux *aux) +{ + u32 moff = aux->attach_st_ops_member_off; + bool wakeup = false; + + if (moff == offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + wakeup = true; + + if (rcvlowat < 0) + rcvlowat = INT_MAX; + + return __tcp_set_rcvlowat(sk, rcvlowat, wakeup); +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_tcp_ops_set_flags_kfunc_set) BTF_ID_FLAGS(func, bpf_tcp_ops_set_flags) BTF_KFUNCS_END(bpf_tcp_ops_set_flags_kfunc_set) +BTF_KFUNCS_START(bpf_tcp_ops_set_rcvlowat_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_rcvlowat, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(bpf_tcp_ops_set_rcvlowat_kfunc_set) + static int bpf_tcp_ops_set_flags_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) { @@ -390,6 +409,31 @@ static const struct btf_kfunc_id_set bpf_tcp_ops_set_flags_kfunc_id_set = { .filter = bpf_tcp_ops_set_flags_kfunc_filter, }; +static int bpf_tcp_ops_set_rcvlowat_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + u32 moff; + + if (!btf_id_set8_contains(&bpf_tcp_ops_set_rcvlowat_kfunc_set, kfunc_id)) + return 0; + + if (prog->aux->st_ops != &bpf_tcp_ops) + return -EACCES; + + moff = prog->aux->attach_st_ops_member_off; + if (moff != offsetof(struct bpf_tcp_ops, enqueue_rcvq) && + moff != offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + return -EACCES; + + return 0; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_rcvlowat_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_rcvlowat_kfunc_set, + .filter = bpf_tcp_ops_set_rcvlowat_kfunc_filter, +}; + static int __init __bpf_tcp_ops_init(void) { int ret; @@ -399,6 +443,8 @@ static int __init __bpf_tcp_ops_init(void) /* BPF_PROG_TYPE_CGROUP_{SOCKOPT,SKB} share BTF_KFUNC_HOOK_CGROUP. */ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCKOPT, &bpf_tcp_ops_set_flags_kfunc_id_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_rcvlowat_kfunc_id_set); ret = ret ?: register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); return ret; From 11f1c0b8a43af7ffc2d65beb77c7f572d8567a61 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Thu, 8 Oct 2026 03:15:30 +0000 Subject: [PATCH 452/660] selftest: bpf: Add test for bpf_tcp_ops.{enqueue,dequeue}_rcvq(). The test is roughly divided into two stages, and the sequence is as follows: I) Setup 1. Attach two BPF programs to a cgroup 2. Establish a TCP connection (@client <-> @child) within the cgroup 3. Enable BPF_TCP_OPS_FLAG_RCVQ on @child via setsockopt() II) RPC frame exchange in various patterns 4. Send a partial RPC descriptor from @client to @child 5. Verify that epoll does NOT wake up @child 6. Send the remaining data of the RPC frame 7. Verify that epoll finally wakes up @child During setup, two BPF programs are attached to simulate a real-world scenario; one is bpf_tcp_ops and the other is CGROUP_SOCKOPT. While the bpf_tcp_ops prog handles the dynamic adjustment of sk->sk_rcvlowat, the CGROUP_SOCKOPT prog is used to enable the TCP AutoLOWAT feature via userspace setsockopt() using pseudo options: #define SOL_BPF 0xdeadbeef #define BPF_TCP_AUTOLOWAT 0x8badf00d setsockopt(fd, SOL_BPF, BPF_TCP_AUTOLOWAT, &(int){1}, sizeof(int)); This reflects a common production use case where an application decides to start parsing RPC frames only at a certain point in the stream (e.g., after HTTP Upgrade), rather than immediately after TCP 3WHS (->passive_established(), etc). When BPF_TCP_AUTOLOWAT is enabled, the BPF prog sets BPF_TCP_OPS_FLAG_RCVQ and initializes sk_local_storage for two sequence numbers to manage its state. Then, for the RPC frame exchange, this test uses a simple format defined as follows: 0 8 16 24 32 +--------+--------+-------+--------+ `. | header size | | +--------+--------+-------+--------+ > RPC descriptor (8 bytes) | payload size | | +--------+--------+-------+--------+ .' ~ header ~ +--------+--------+-------+--------+ ~ payload ~ +--------+--------+-------+--------+ Every time a new skb is enqueued to sk->sk_receive_queue, the bpf_tcp_ops prog parses it and updates these sequence numbers: rpc_desc_seq : the SEQ # of the start of the RPC descriptor rpc_end_seq : the SEQ # of the end of the RPC frame => rpc_desc_seq + 8 + header size + payload size Assume we receive two RPC descriptors in the following pattern: 1. When we receive skb-1, only part of the RPC descriptor is parsed. rpc_desc_seq is set to the first byte while rpc_end_seq is unknown. Thus, sk->sk_rcvlowat is set to the size of the RPC descriptor (8 bytes). <- skb-1 -> <---- skb-2 ----> <------ skb-3 -----> +-----------+.................+....................+...... | RPC desc 1 | header + payload | RPC desc 2 | ... +-----------+.................+....................+...... ^ ^-. `- rpc_desc_seq `- sk->sk_rcvlowat 2. Next, we receive skb-2, which completes the first RPC descriptor. Now rpc_end_seq is known, so sk->sk_rcvlowat is advanced to it. <- skb-1 -> <---- skb-2 ----> <------ skb-3 -----> +-----------+-----------------+....................+...... | RPC desc 1 | header + payload | RPC desc 2 | ... +-----------+-----------------+....................+...... ^ ^ '- rpc_desc_seq '- rpc_end_seq & sk->sk_rcvlowat 3. Once we receive skb-3, which contains the next full RPC descriptor, rpc_desc_seq is advanced and rpc_end_seq is updated according to the size of RPC frame 2. Note that sk->sk_rcvlowat is NOT updated to the new rpc_end_seq yet. This ensures that the application is woken up to read the already complete RPC frame 1. <- skb-1 -> <---- skb-2 ----> <------ skb-3 -----> +-----------+-----------------+--------------------+...... | RPC desc 1 | header + payload | RPC desc 2 | ... | +-----------+-----------------+--------------------+...... ^ ^ rpc_desc_seq -----------' rpc_end_seq ----...-' & sk->sk_rcvlowat This sequence corresponds to the 4th test case in rpc_test_cases[], and we can see helpful output if we "#define DEBUG": # cat /sys/kernel/tracing/trace_pipe | \ awk '{ if ($0 ~ /AF_/) sub(/^.*AF_/, "AF_"); print $0 }' & \ BGPID=$!; ./test_progs -t tcp_autolowat; kill -9 -$BGPID ... AF_INET6 rpc_test_cases[3]: Start parsing skb: seq: 0, end_seq: 1, len: 1, rpc_desc_seq: 0, rpc_end_seq: 0, rpc_buff_len: 0 AF_INET6 rpc_test_cases[3]: Copied 1 bytes: rpc_desc_buff_len: 1 AF_INET6 rpc_test_cases[3]: Setting rcvlowat: tp->copied_seq: 0, rpc_desc_seq: 0, rpc_end_seq: 0, rpc_desc_buff_len: 1 AF_INET6 rpc_test_cases[3]: Set rcvlowat: expected: 8, actual: 8 AF_INET6 rpc_test_cases[3]: Start parsing skb: seq: 1, end_seq: 8, len: 7, rpc_desc_seq: 0, rpc_end_seq: 0, rpc_buff_len: 1 AF_INET6 rpc_test_cases[3]: Copied full descriptor: rpc_desc_seq: 0, rpc_end_seq: 258, header_len: 100, payload_len: 150 AF_INET6 rpc_test_cases[3]: No more descriptor: rpc_end_seq: 258, end_seq: 8 AF_INET6 rpc_test_cases[3]: Setting rcvlowat: tp->copied_seq: 0, rpc_desc_seq: 0, rpc_end_seq: 258, rpc_desc_buff_len: 8 AF_INET6 rpc_test_cases[3]: Set rcvlowat: expected: 258, actual: 258 ... Signed-off-by: Kuniyuki Iwashima Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Acked-by: Stanislav Fomichev Link: https://patch.msgid.link/20261008031604.256498-11-kuniyu@google.com --- .../selftests/bpf/prog_tests/tcp_autolowat.c | 350 ++++++++++++++++++ .../selftests/bpf/progs/bpf_tracing_net.h | 2 + .../selftests/bpf/progs/tcp_autolowat.c | 294 +++++++++++++++ 3 files changed, 646 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c create mode 100644 tools/testing/selftests/bpf/progs/tcp_autolowat.c diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c new file mode 100644 index 000000000000..337f9d34a39c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include + +#include "test_progs.h" +#include "cgroup_helpers.h" +#include "network_helpers.h" + +#include "tcp_autolowat.skel.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +enum rpc_event_type { + RPC_EVENT_END, + RPC_EVENT_AUTOLOWAT, + RPC_EVENT_SEND, + RPC_EVENT_RECV, + RPC_EVENT_EPOLL, + RPC_EVENT_RCVLOWAT, +}; + +struct rpc_event { + enum rpc_event_type type; + union { + int len; + int nfds; + int val; + int rcvlowat; + }; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) + +static struct rpc_test_case { + char data[4096]; + struct rpc_descriptor desc[32]; + struct rpc_event event[32]; +} rpc_test_cases[] = { + { + .desc = { + { .header_len = 100, .payload_len = 150 }, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 100 + 150}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 3}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Only the next descriptor in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 200, .payload_len = 500}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* The first descriptor is partial. */ + {.type = RPC_EVENT_SEND, .len = 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The first descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* The first header is ready. */ + {.type = RPC_EVENT_SEND, .len = 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* skb has the first payload and 1 byte of the next descriptor. */ + {.type = RPC_EVENT_SEND, .len = 150 + 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* After reading the first RPC message, SO_RCVLOWAT should be RPC_DESC_SIZE. */ + {.type = RPC_EVENT_RECV, .len = RPC_DESC_SIZE + 150 + 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The second descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 200 + 500}, + }, + }, +}; + +struct tcp_autolowat_test_cb { + int saved_netns; + union { + int fd[4]; + struct { + int server, client, child; + int epoll; + }; + }; +}; + +static void tcp_autolowat_teardown_cb(struct tcp_autolowat_test_cb *cb) +{ + int i, err; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) { + if (cb->fd[i] != -1) + close(cb->fd[i]); + } + + if (cb->saved_netns != -1) { + err = setns(cb->saved_netns, CLONE_NEWNET); + ASSERT_OK(err, "restore netns"); + + close(cb->saved_netns); + } +} + +static int tcp_autolowat_setup_cb(struct tcp_autolowat_test_cb *cb, int family) +{ + struct epoll_event ev = {}; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) + cb->fd[i] = -1; + + cb->saved_netns = open("/proc/self/ns/net", O_RDONLY); + if (!ASSERT_OK_FD(cb->saved_netns, "save netns")) + goto err; + + err = unshare(CLONE_NEWNET); + if (!ASSERT_OK(err, "unshare")) + goto err; + + err = system("ip link set dev lo up"); + if (!ASSERT_OK(err, "set up lo")) + goto err; + + cb->server = start_server(family, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_OK_FD(cb->server, "start_server")) + goto err; + + cb->client = connect_to_fd(cb->server, 0); + if (!ASSERT_OK_FD(cb->client, "connect_to_fd")) + goto err; + + cb->child = accept(cb->server, NULL, NULL); + if (!ASSERT_OK_FD(cb->child, "accept")) + goto err; + + cb->epoll = epoll_create1(0); + if (!ASSERT_OK_FD(cb->epoll, "epoll_create")) + goto err; + + ev.events = EPOLLIN; + ev.data.fd = cb->child; + + err = epoll_ctl(cb->epoll, EPOLL_CTL_ADD, cb->child, &ev); + if (!ASSERT_OK(err, "epoll_ctl")) + goto err; + + return 0; + +err: + tcp_autolowat_teardown_cb(cb); + return -1; +} + +static int tcp_autolowat_build_data(struct rpc_test_case *test_case) +{ + struct rpc_descriptor *desc = test_case->desc; + char *ptr = test_case->data; + int rpc_size; + + memset(ptr, 0, sizeof(test_case->data)); + + while (desc->header_len + desc->payload_len) { + rpc_size = sizeof(*desc) + desc->header_len + desc->payload_len; + + if (!ASSERT_LE(ptr + rpc_size - test_case->data, + sizeof(test_case->data), "data overflow")) + return 1; + + memcpy(ptr, desc, sizeof(*desc)); + ptr += rpc_size; + desc++; + } + + if (!ASSERT_GT(ptr - test_case->data, 0, "no data")) + return 1; + + return 0; +} + +static void tcp_autolowat_run_rpc_test(struct tcp_autolowat_test_cb *cb, + struct rpc_test_case *test_case) +{ + struct rpc_event *event = test_case->event; + char *ptr = test_case->data; + struct epoll_event ev; + socklen_t optlen; + int err, optval; + char buf[4096]; + + if (tcp_autolowat_build_data(test_case)) + return; + + while (1) { + switch (event->type) { + case RPC_EVENT_END: + return; + case RPC_EVENT_AUTOLOWAT: + err = setsockopt(cb->child, SOL_BPF, BPF_TCP_AUTOLOWAT, + &event->val, sizeof(event->val)); + if (!ASSERT_OK(err, "setsockopt")) + return; + break; + case RPC_EVENT_SEND: + err = send(cb->client, ptr, event->len, 0); + if (!ASSERT_EQ(err, event->len, "send")) + return; + + ptr += event->len; + break; + case RPC_EVENT_RECV: + err = recv(cb->child, buf, event->len, 0); + if (!ASSERT_EQ(err, event->len, "recv")) + return; + break; + case RPC_EVENT_EPOLL: + err = epoll_wait(cb->epoll, &ev, 1, 100); + if (!ASSERT_EQ(err, event->nfds, "epoll_wait")) + return; + break; + case RPC_EVENT_RCVLOWAT: + optval = 0; + optlen = sizeof(optval); + + err = getsockopt(cb->child, SOL_SOCKET, SO_RCVLOWAT, &optval, &optlen); + if (!ASSERT_OK(err, "getsockopt") || + !ASSERT_EQ(optval, event->rcvlowat, "rcvlowat")) + return; + break; + } + + event++; + } +} + +static void tcp_autolowat_run_rpc_tests(struct tcp_autolowat *skel, int family) +{ + struct tcp_autolowat_test_cb cb; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(rpc_test_cases); i++) { + memset(skel->bss->test_name, 0, sizeof(skel->bss->test_name)); + + snprintf(skel->bss->test_name, sizeof(skel->bss->test_name), + "AF_INET%c rpc_test_cases[%d]", + family == AF_INET ? ' ' : '6', i); + + if (!test__start_subtest(skel->bss->test_name)) + continue; + + err = tcp_autolowat_setup_cb(&cb, family); + if (err) + continue; + + tcp_autolowat_run_rpc_test(&cb, &rpc_test_cases[i]); + tcp_autolowat_teardown_cb(&cb); + } +} + +static void tcp_autolowat_run_tests(struct tcp_autolowat *skel) +{ + tcp_autolowat_run_rpc_tests(skel, AF_INET); + tcp_autolowat_run_rpc_tests(skel, AF_INET6); +} + +void test_tcp_autolowat(void) +{ + struct tcp_autolowat *skel; + struct bpf_link *link[2]; + int cgroup; + + skel = tcp_autolowat__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + cgroup = test__join_cgroup("/tcp_autolowat"); + if (!ASSERT_GE(cgroup, 0, "join_cgroup")) + goto destroy_skel; + + link[0] = bpf_map__attach_cgroup_opts(skel->maps.tcp_autolowat_ops, cgroup, NULL); + if (!ASSERT_OK_PTR(link[0], "attach_cgroup(tcp_autolowat_ops)")) + goto close_cgroup; + + link[1] = bpf_program__attach_cgroup(skel->progs.tcp_autolowat_setsockopt, cgroup); + if (!ASSERT_OK_PTR(link[1], "attach_cgroup(SETSOCKOPT)")) + goto destroy_sockops; + + tcp_autolowat_run_tests(skel); + + bpf_link__destroy(link[1]); +destroy_sockops: + bpf_link__destroy(link[0]); +close_cgroup: + close(cgroup); +destroy_skel: + tcp_autolowat__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 593b38f90417..4c999d59cbbc 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -79,6 +79,8 @@ #define NEXTHDR_TCP 6 +#define TCPHDR_FIN 0x01 + #define TCPOPT_NOP 1 #define TCPOPT_EOL 0 #define TCPOPT_MSS 2 diff --git a/tools/testing/selftests/bpf/progs/tcp_autolowat.c b/tools/testing/selftests/bpf/progs/tcp_autolowat.c new file mode 100644 index 000000000000..8a55f3cee260 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_autolowat.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include "vmlinux.h" + +#include +#include +#include +#include +#include + +#include "bpf_tracing_net.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +//#define DEBUG /* For verbose output. */ + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) +#define MAX_RPC_DESC_PER_SKB 100 + +struct tcp_autolowat_cb { + /* Don't put this field at the end; BPF verifier complains. */ + char rpc_desc_buf[RPC_DESC_SIZE]; + u32 rpc_desc_seq; + u32 rpc_end_seq; +#ifdef DEBUG + u32 isn; +#endif + u8 rpc_desc_buff_len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tcp_autolowat_cb); +} tcp_autolowat_map SEC(".maps"); + +char test_name[64]; + +#ifdef DEBUG +#define LOG(str, ...) \ + bpf_printk("%s: " str, test_name, ##__VA_ARGS__) +#else +#define LOG(...) +#endif + +#define SEQ(val) \ + (val - cb->isn) +#define TP_SEQ(field) \ + (tp->field - cb->isn) +#define CB_SEQ(field) \ + (cb->field - cb->isn) + +static int tcp_parse_descriptor(struct tcp_autolowat_cb *cb, + struct bpf_dynptr *dptr, + u32 seq, u32 end_seq) +{ + struct rpc_descriptor *rpc_desc; + u32 rpc_copied_seq; + u64 copy_len; /* u32 should work, but not for no_alu32 :/ */ + u64 rpc_len; + int err; + + rpc_copied_seq = cb->rpc_desc_seq + cb->rpc_desc_buff_len; + + if (before(cb->rpc_desc_seq + RPC_DESC_SIZE, end_seq)) + copy_len = RPC_DESC_SIZE - cb->rpc_desc_buff_len; + else + copy_len = end_seq - rpc_copied_seq; + + if (copy_len == 0) + goto disable; /* FIN. */ + if (copy_len > RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + if (cb->rpc_desc_buff_len >= RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + + err = bpf_dynptr_read(cb->rpc_desc_buf + cb->rpc_desc_buff_len, + copy_len, dptr, rpc_copied_seq - seq, 0); + if (err) + goto disable; + + cb->rpc_desc_buff_len += copy_len; + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + LOG("Copied %d bytes: rpc_desc_buff_len: %u", copy_len, cb->rpc_desc_buff_len); + goto partial; + } + + rpc_desc = (struct rpc_descriptor *)cb->rpc_desc_buf; + rpc_len = RPC_DESC_SIZE + rpc_desc->header_len + rpc_desc->payload_len; + + if (rpc_len > INT_MAX) + goto disable; + + cb->rpc_end_seq = cb->rpc_desc_seq + rpc_len; + + LOG("Copied full descriptor: rpc_desc_seq: %u, rpc_end_seq: %u, header_len: %u, payload_len: %u", + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), + rpc_desc->header_len, rpc_desc->payload_len); + + return 0; +disable: + return -1; +partial: + return 1; +} + +static void tcp_set_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk) +{ + struct tcp_sock *tp = (struct tcp_sock *)sk; + u32 val; /* To handle wraparound. */ + + LOG("Setting rcvlowat: tp->copied_seq: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_desc_buff_len: %u", + TP_SEQ(copied_seq), CB_SEQ(rpc_desc_seq), + CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (before(tp->copied_seq, cb->rpc_desc_seq)) + val = cb->rpc_desc_seq - tp->copied_seq; + else if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) + val = RPC_DESC_SIZE; + else + val = cb->rpc_end_seq - tp->copied_seq; + + if (val != tp->inet_conn.icsk_inet.sk.sk_rcvlowat) { + bpf_tcp_ops_set_rcvlowat(sk, val); + + LOG("Set rcvlowat: expected: %u, actual: %d\n", + val, tp->inet_conn.icsk_inet.sk.sk_rcvlowat); + } else { + LOG("No need to set rcvlowat: %u\n", val); + } +} + +static void tcp_disable_autolowat(struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, 0, BPF_TCP_OPS_FLAG_RCVQ); + + bpf_tcp_ops_set_rcvlowat(sk, 1); + + LOG("Disabled autolowat"); +} + +static void tcp_do_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk, struct sk_buff *skb) +{ + struct bpf_dynptr dptr; + struct tcp_skb_cb *tcb; + u32 seq, end_seq; + int ret = 0, i; + + if (bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &dptr)) { + ret = -1; + goto update; + } + + tcb = bpf_core_cast(skb->cb, struct tcp_skb_cb); + seq = tcb->seq; + end_seq = tcb->end_seq - !!(tcb->tcp_flags & TCPHDR_FIN); + + LOG("Start parsing skb: seq: %u, end_seq: %u, len: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_buff_len: %u", + SEQ(seq), SEQ(end_seq), end_seq - seq, + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + goto update; + } + + i = 0; + + while (1) { + if (i++ > MAX_RPC_DESC_PER_SKB) { + ret = -1; + break; + } + + if (after(cb->rpc_end_seq, end_seq)) { + LOG("No more descriptor: rpc_end_seq: %u, end_seq: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq)); + break; + } + + cb->rpc_desc_seq = cb->rpc_end_seq; + cb->rpc_desc_buff_len = 0; + + if (cb->rpc_end_seq == end_seq) + break; + + LOG("Found next descriptor: rpc_end_seq: %u, end_seq: %u, len: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq), end_seq - cb->rpc_end_seq); + + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + break; + } + +update: + if (ret >= 0) + tcp_set_autolowat(cb, sk); + else + tcp_disable_autolowat(sk); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_enqueue_rcvq, struct sock *sk, struct sk_buff *skb) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_do_autolowat(cb, sk, skb); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_dequeue_rcvq, struct sock *sk) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_set_autolowat(cb, sk); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_autolowat_ops = { + .enqueue_rcvq = (void *)tcp_autolowat_enqueue_rcvq, + .dequeue_rcvq = (void *)tcp_autolowat_dequeue_rcvq, +}; + +static int tcp_init_autolowat_cb(struct bpf_tcp_sock *btp) +{ + struct tcp_autolowat_cb *cb; + struct tcp_sock *tp; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, btp, 0, + BPF_SK_STORAGE_GET_F_CREATE); + if (!cb) + return -1; + + tp = bpf_core_cast(btp, struct tcp_sock); + + cb->rpc_desc_seq = tp->copied_seq; + cb->rpc_end_seq = tp->copied_seq; +#ifdef DEBUG + cb->isn = tp->copied_seq; +#endif + + return bpf_tcp_ops_set_flags((struct tcp_sock *)btp, + BPF_TCP_OPS_FLAG_RCVQ, 0); +} + +SEC("cgroup/setsockopt") +int tcp_autolowat_setsockopt(struct bpf_sockopt *ctx) +{ + void *optval_end = ctx->optval_end; + int *optval = ctx->optval; + struct bpf_tcp_sock *btp; + + if (ctx->level != SOL_BPF || ctx->optname != BPF_TCP_AUTOLOWAT) + goto out; + + if (optval + 1 > optval_end) + return 0; /* -EPERM */ + + btp = bpf_tcp_sock(ctx->sk); + if (!btp) + goto out; + + if (*optval && tcp_init_autolowat_cb(btp)) + return 0; /* -EPERM */ + + /* + * BPF has consumed this option, don't call kernel + * setsockopt handler. + */ + ctx->optlen = -1; +out: + return 1; +} + +char _license[] SEC("license") = "GPL"; From 4cce28612bf7811c326b99a92e19eef0e3f81b28 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Sat, 10 Oct 2026 13:58:26 -0700 Subject: [PATCH 453/660] bpf: Use div64_u64{,_rem}() in scev.c For whatever reason gcc built-ins implementing 64-bit division are not wired into 32-bit kernel builds: ld: kernel/bpf/scev.c:2074:(.text+0x4512): undefined reference to `__udivdi3' ld: kernel/bpf/scev.c:2141:(.text+0x475b): undefined reference to `__udivmoddi4' Instead, one has to use div64_u64() and div64_u64_rem() explicitly. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Reported-by: Tejun Heo Reported-by: kernel test robot Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20261010205828.3366892-1-eddyz87@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/scev.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index adad32d8cb12..48500d73105a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -2083,7 +2084,7 @@ static bool compute_max_iters(struct bpf_verifier_env *env, struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop; struct scev *scev = env->scev; struct scev_value initial_val, step_val, bound_val; - u64 step, diff, bound, initial, backedge_taken_count; + u64 rem, step, diff, bound, initial, backedge_taken_count; u8 op = latch->op; bool up, down; @@ -2129,26 +2130,26 @@ static bool compute_max_iters(struct bpf_verifier_env *env, /* Is the latch condition false on the first iteration? */ initial >= bound || /* Count iterations satisfying the strict inequality, including i=0. */ - check_add_overflow((diff - 1) / step, 1, &backedge_taken_count) || + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || /* * Check if the last iteration overflows the counter: * initial + backedge_taken_count * step > U64_MAX */ - backedge_taken_count > (U64_MAX - initial) / step) + backedge_taken_count > div64_u64(U64_MAX - initial, step)) return false; break; case BPF_JLE: /* E.g.: 3 + 2*i ≤ 7 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ if (down || initial > bound || /* Count iterations satisfying the non-strict inequality. */ - check_add_overflow(diff / step, 1, &backedge_taken_count) || - backedge_taken_count > (U64_MAX - initial) / step) + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(U64_MAX - initial, step)) return false; break; case BPF_JGT: /* E.g.: 7 - 2*i > 3 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. */ if (up || initial <= bound || - check_add_overflow((-diff - 1) / -step, 1, &backedge_taken_count) || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || /* * The last iteration underflows the counter if: * initial + backedge_taken_count * step < 0 ⇔ @@ -2156,35 +2157,35 @@ static bool compute_max_iters(struct bpf_verifier_env *env, * backedge_taken_count > initial / -step * (direction flips because step < 0) */ - backedge_taken_count > initial / -step) + backedge_taken_count > div64_u64(initial, -step)) return false; break; case BPF_JGE: /* E.g.: 7 - 2*i ≥ 3 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ if (up || initial < bound || - check_add_overflow(-diff / -step, 1, &backedge_taken_count) || - backedge_taken_count > initial / -step) + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial, -step)) return false; break; case BPF_JSLT: if (down || (s64)initial >= (s64)bound || - check_add_overflow((diff - 1) / step, 1, &backedge_taken_count) || + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || /* * Check if the last iteration overflows the signed counter: * (s64)initial + backedge_taken_count * step > S64_MAX */ - backedge_taken_count > ((u64)S64_MAX - initial) / step) + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) return false; break; case BPF_JSLE: if (down || (s64)initial > (s64)bound || - check_add_overflow(diff / step, 1, &backedge_taken_count) || - backedge_taken_count > ((u64)S64_MAX - initial) / step) + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) return false; break; case BPF_JSGT: if (up || (s64)initial <= (s64)bound || - check_add_overflow((-diff - 1) / -step, 1, &backedge_taken_count) || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || /* * The last iteration underflows the signed counter if: * (s64)initial + backedge_taken_count * step < S64_MIN ⇔ @@ -2192,13 +2193,13 @@ static bool compute_max_iters(struct bpf_verifier_env *env, * backedge_taken_count > initial - (u64)S64_MIN / -step * (direction flips because step < 0) */ - backedge_taken_count > (initial - (u64)S64_MIN) / -step) + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) return false; break; case BPF_JSGE: if (up || (s64)initial < (s64)bound || - check_add_overflow(-diff / -step, 1, &backedge_taken_count) || - backedge_taken_count > (initial - (u64)S64_MIN) / -step) + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) return false; break; case BPF_JNE: @@ -2208,9 +2209,9 @@ static bool compute_max_iters(struct bpf_verifier_env *env, */ diff = down ? -diff : diff; step = down ? -step : step; - if (diff == 0 || diff % step) + backedge_taken_count = div64_u64_rem(diff, step, &rem); + if (diff == 0 || rem != 0) return false; - backedge_taken_count = diff / step; break; default: return false; From 10aaef818b15c240af169df6b7f4a619f38e85ae Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 10:46:59 +0000 Subject: [PATCH 454/660] MAINTAINERS: Add myself as co-maintainer of riscv bpf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Over the years, I have been actively involved in the development and have proactively taken on maintenance responsibilities for the RV64 JIT, covering the related bpf core, riscv architecture, and selftests/CI infrastructure. With the continuous growth of riscv bpf features and testing workload, adding a co-maintainer helps share the review burden and ensure timely patch handling. As for the RV32 JIT, there has been a lack of active maintainer response for several years, leaving incoming patches occasionally stalled. Active maintenance is needed to review incoming patches and ensure issues are addressed in a timely manner. Therefore, I would like to request taking on the co-maintainer role for both the RV64 and RV32 JITs. For me, being a maintainer is both an honor and a responsibility, and it also makes it easier for me to participate in riscv bpf work, where I look forward to dedicating more time and effort. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Acked-by: Björn Töpel Link: https://patch.msgid.link/20260930104659.1919027-1-pulehui@huaweicloud.com --- MAINTAINERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index ac037cbd4e21..fe99b7437a2d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4872,6 +4872,7 @@ F: arch/powerpc/net/ BPF JIT for RISC-V (32-bit) M: Luke Nelson M: Xi Wang +M: Pu Lehui L: bpf@vger.kernel.org S: Maintained F: arch/riscv/net/ @@ -4879,7 +4880,7 @@ X: arch/riscv/net/bpf_jit_comp64.c BPF JIT for RISC-V (64-bit) M: Björn Töpel -R: Pu Lehui +M: Pu Lehui R: Puranjay Mohan L: bpf@vger.kernel.org S: Maintained From 1cec0b70fdc678d6c0252f3f25e23df0b5a5e49c Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 06:49:27 +0000 Subject: [PATCH 455/660] riscv, bpf: Support indirect calls to bpf subprogs (callx) Implement callx by emitting a jalr instruction to jump to the target subprog address held in rd, and marking the call as it clobbers RV_REG_RA. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930064928.1909085-2-pulehui@huaweicloud.com --- arch/riscv/net/bpf_jit_comp64.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 26da34387e03..ebd6873b9b3d 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1921,6 +1921,13 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; } + + /* indirect call to a bpf subprog via the address in rd */ + case BPF_JMP | BPF_CALL | BPF_X: + mark_call(ctx); + emit_jalr(RV_REG_RA, rd, 0, ctx); + break; + /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(i, ctx)) @@ -2280,3 +2287,8 @@ bool bpf_jit_supports_stack_args(void) { return true; } + +bool bpf_jit_supports_callx(void) +{ + return true; +} From 530dfafc61075098046e5bb042d8366120f3cc32 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 06:49:28 +0000 Subject: [PATCH 456/660] selftests/bpf: Enable callx related testcases for RV64 Enable callx bpf selftests for RV64 now that the RV64 JIT supports indirect calls to subprogs. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930064928.1909085-3-pulehui@huaweicloud.com --- tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c | 3 ++- tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_callx.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_callx_rodata.c | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c index 42b33d5876b1..cc63b84e4d94 100644 --- a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c @@ -8,7 +8,8 @@ #include #include -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) #define CALLEE_INSN 6 #define DATA 0x1234 diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c index fc5181c5b5ba..f6ef67c3141e 100644 --- a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -3,7 +3,8 @@ #include "callx_rodata.lskel.h" -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) static void run(int prog_fd, int expected, const char *name) { diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c index bc31d0221f6a..7e59188d9fca 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -7,7 +7,8 @@ #include "../../../include/linux/filter.h" /* gcc doesn't support indirect calls */ -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && defined(__clang__) +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) #define CALLX_INSN(DST, SRC, OFF, IMM) \ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c index 3e7a1c2c2a9a..6cf58ca9ac5b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -7,7 +7,8 @@ #include "../../../include/linux/filter.h" /* gcc doesn't support indirect calls */ -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && defined(__clang__) +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) /* * Read-only data with pointers to functions, where the compiler puts tables From b4a1ecae94ae803fcd75fb16a7653ff86b64ed8b Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Wed, 30 Sep 2026 08:35:19 +0000 Subject: [PATCH 457/660] bpf, riscv: Fix stack-passed arguments for indirect trampolines store_args() reads stack-passed arguments relative to FP assuming the trampoline was entered through the fentry call from a traced function. In that path, the trampoline pushes the parent frame before establishing its final FP, so the incoming stack arguments start at FP + 16. An indirect trampoline for a struct_ops callback is called through a function pointer. Its prologue allocates only the trampoline frame and sets FP to the incoming SP. The RISC-V ABI places the first stack argument at that incoming SP, so the arguments start at FP, not FP + 16. Every stack-passed argument of a callback with more than eight argument slots is therefore read two slots late. Pass the prologue-dependent offset to store_args(), using zero for a direct struct_ops trampoline and 16 for the fentry path. Fixes: 6801b0aef79d ("riscv, bpf: Add 12-argument support for RV64 bpf trampoline") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930083522.1914145-2-pulehui@huaweicloud.com --- arch/riscv/net/bpf_jit_comp64.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index ebd6873b9b3d..016994aef94d 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -913,7 +913,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } -static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) +static void store_args(int nr_arg_slots, int args_off, int stack_args_off, + struct rv_jit_context *ctx) { int i; @@ -921,8 +922,8 @@ static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ct if (i < RV_MAX_REG_ARGS) { emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); } else { - /* skip slots for T0 and FP of traced function */ - emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + emit_ld(RV_REG_T1, stack_args_off + + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); } args_off -= 8; @@ -1217,7 +1218,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, func_meta = nr_arg_slots; emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); - store_args(nr_arg_slots, args_off, ctx); + /* + * A direct struct_ops call has its first stack argument at the incoming + * SP, which the trampoline keeps as FP. The fentry path pushes the + * parent frame first, so its incoming stack arguments start at FP + 16. + */ + store_args(nr_arg_slots, args_off, is_struct_ops ? 0 : 16, ctx); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ From 153a855d0b3d3b740d7470fc248fe4ebccf06970 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 08:35:20 +0000 Subject: [PATCH 458/660] selftests/bpf: Avoid jiffies quantization error in setget_sockopt The setget_sockopt selftest fails on kernels with HZ=250 (such as the default on riscv64), specifically in the TCP_BPF_DELACK_MAX and TCP_BPF_RTO_MIN sub-tests. The test sets a socket option value in microseconds via bpf_setsockopt() and immediately reads it back via bpf_getsockopt() to verify that it matches. However, the kernel converts the value to jiffies internally for storage and converts it back to microseconds upon retrieval. When HZ=250 (4000 us per jiffy), 30000 us cannot be divided evenly: usecs_to_jiffies(30000) -> 8 jiffies jiffies_to_usecs(8) -> 32000 us (!= 30000 us) This mismatch triggers an assertion failure. Change the test value from 30000 to 40000. 40000 us divides evenly across common kernel timer frequencies (HZ=100, HZ=250, and HZ=1000), avoiding quantization errors and making the test robust across different architectures. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930083522.1914145-3-pulehui@huaweicloud.com --- tools/testing/selftests/bpf/progs/setget_sockopt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index d96e99b67aeb..95e6a337ab71 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -61,8 +61,8 @@ static const struct sockopt_test sol_tcp_tests[] = { { .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, }, { .opt = TCP_BPF_SOCK_OPS_CB_FLAGS, .new = BPF_SOCK_OPS_ALL_CB_FLAGS, .expected = BPF_SOCK_OPS_ALL_CB_FLAGS, }, - { .opt = TCP_BPF_DELACK_MAX, .new = 30000, .expected = 30000, }, - { .opt = TCP_BPF_RTO_MIN, .new = 30000, .expected = 30000, }, + { .opt = TCP_BPF_DELACK_MAX, .new = 40000, .expected = 40000, }, + { .opt = TCP_BPF_RTO_MIN, .new = 40000, .expected = 40000, }, { .opt = TCP_RTO_MAX_MS, .new = 2000, .expected = 2000, }, { .opt = 0, }, }; From 78142aa07ef69a314d3da2309c3ed6f024ec915e Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 08:35:21 +0000 Subject: [PATCH 459/660] selftests/bpf: Update qemu config for riscv64 in vmtest.sh When running vmtest.sh on riscv64, the kernel panic at boot due to an illegal instruction. This is because '-cpu rv64' only enables a baseline set of riscv extensions, whereas the kernel or toolchain requires newer extensions. Switch to '-cpu max' so qemu enables all supported extensions, and bump the required qemu version accordingly. Additionally, qemu's pmu emulation for riscv64 currently has issues, causing perf_event_open-related tests to fail. Temporarily disable the sscofpmf extension to bypass these failures for now, and will try to investigate and address it later. While at it, update the libbpf/ci branch in INDEX_URL from 'master' to 'main'. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260930083522.1914145-4-pulehui@huaweicloud.com --- tools/testing/selftests/bpf/vmtest.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index e7e0b419a0b8..c4663cad74f5 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -35,11 +35,11 @@ aarch64) ARCH="arm64" ;; riscv64) - # required qemu version v7.2.0+ + # required qemu version v8.2.0+ QEMU_BINARY=qemu-system-riscv64 QEMU_CONSOLE="ttyS0,115200" HOST_FLAGS=(-M virt -cpu host -enable-kvm -smp 8) - CROSS_FLAGS=(-M virt -cpu rv64,sscofpmf=true -smp 8) + CROSS_FLAGS=(-M virt -cpu max,sscofpmf=false -smp 8) BZIMAGE="arch/riscv/boot/Image" ARCH="riscv" ;; @@ -65,7 +65,7 @@ OUTPUT_DIR="$HOME/.bpf_selftests" KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config" "tools/testing/selftests/bpf/config.vm" "tools/testing/selftests/bpf/config.${PLATFORM}") -INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX" +INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/main/INDEX" NUM_COMPILE_JOBS="$(nproc)" LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")" LOG_FILE="${LOG_FILE_BASE}.log" From 52e98bc77f32d6a485781f0d666728bc5785473f Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 30 Sep 2026 08:35:22 +0000 Subject: [PATCH 460/660] selftests/bpf: Update DENYLIST.riscv64 Both probe_user and stacktrace_build_id fail on riscv64 due to riscv64's trap-based kprobe handling. For probe_user, kprobes on riscv64 are handled via traps where the entry path invokes irqentry_nmi_enter(). Because bpf_probe_write_user() rejects execution in NMI context, the test fails. Upstream discussions regarding NMI semantics on riscv are ongoing [0] and will be tracked. For stacktrace_build_id, under the kprobe trap context with interrupts disabled, bpf_get_stackid() obtains a valid build_id, but the queued irq_work is deferred. As a result, subsequent bpf_get_stack() fails to acquire mmap_read_trylock() and falls back to BPF_STACK_BUILD_ID_IP, causing an inconsistency between the two results. Since stacktrace_build_id is already included in the generic DENYLIST, only add probe_user to DENYLIST.riscv64. Signed-off-by: Pu Lehui Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/linux-riscv/20230702025708.784106-1-guoren@kernel.org [0] Link: https://patch.msgid.link/20260930083522.1914145-5-pulehui@huaweicloud.com --- tools/testing/selftests/bpf/DENYLIST.riscv64 | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index ca1beae7fe8f..75a9690fb2f2 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,2 +1,3 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions +probe_user # kprobe hit in trap From c0316386fecd0633b08ed87fb707eb42dc900dfe Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Mon, 28 Sep 2026 15:18:42 +0800 Subject: [PATCH 461/660] bpf, riscv: Add support for indirect jumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement JIT support for the indirect jump instruction (BPF_JMP | BPF_JA | BPF_X), a.k.a. gotox, which lets a BPF program jump through a BPF_MAP_TYPE_INSN_ARRAY jump table. Emit "jalr zero, rd, 0" and hand the xlated to jitted offsets to bpf_prog_update_insn_ptrs(), which is what fills in the jump table entries; without that call the load fails with -EFAULT in bpf_insn_array_ready(). ctx->offset[] holds the offset of the insn *following* insn i, as bpf_prog_fill_jited_linfo() expects, so it is shifted by one and offset[0] comes from the prologue length. build_body() now records both halves of a multi-insn record, so no slot keeps a fabricated offset. Only the RV64 JIT is covered; RV32 keeps failing to load as before. Signed-off-by: Chen Pei Signed-off-by: Alexei Starovoitov Reviewed-by: Björn Töpel Reviewed-by: Pu Lehui Acked-by: Björn Töpel Link: https://patch.msgid.link/20260928071845.2432-2-cp0613@linux.alibaba.com --- arch/riscv/net/bpf_jit_comp64.c | 5 +++++ arch/riscv/net/bpf_jit_core.c | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 016994aef94d..788fcc21fd5b 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1719,6 +1719,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zextw(rd, rd, ctx); break; + /* JUMP reg */ + case BPF_JMP | BPF_JA | BPF_X: + emit_jalr(RV_REG_ZERO, rd, 0, ctx); + break; + /* JUMP off */ case BPF_JMP | BPF_JA: case BPF_JMP32 | BPF_JA: diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 470a6ace5662..5265dd5bd39e 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -26,10 +26,13 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) int ret; ret = bpf_jit_emit_insn(insn, ctx, extra_pass); - if (ret > 0) - i++; /* skip the next instruction */ if (offset) offset[i] = ctx->ninsns; + if (ret > 0) { + i++; /* skip the next instruction */ + if (offset) + offset[i] = ctx->ninsns; + } if (ret < 0) return ret; } @@ -176,6 +179,15 @@ skip_init_ctx: for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); + + /* + * bpf_prog_update_insn_ptrs() wants the start of each insn, so + * shift ctx->offset[] by one and get insn 0 from the prologue. + */ + for (i = prog->len - 1; i > 0; i--) + ctx->offset[i] = ctx->offset[i - 1]; + ctx->offset[0] = ninsns_rvoff(ctx->prologue_len); + bpf_prog_update_insn_ptrs(prog, ctx->offset, jit_data->ro_image); out_offset: kvfree(ctx->offset); kfree(jit_data); From 5f750ad6d67508d83d0aadaae674f4d768814c12 Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Mon, 28 Sep 2026 15:18:43 +0800 Subject: [PATCH 462/660] selftests/bpf: Enable gotox tests for riscv64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The riscv64 JIT now supports the gotox instruction and jump tables, so run the tests in verifier_gotox.c on riscv64 too, mirroring what was done for arm64 and powerpc. The guard is 64-bit only because the RV32 JIT does not implement gotox. Signed-off-by: Chen Pei Signed-off-by: Alexei Starovoitov Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://patch.msgid.link/20260928071845.2432-3-cp0613@linux.alibaba.com --- tools/testing/selftests/bpf/progs/verifier_gotox.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 84302a122ccf..3d1ff7078a72 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -6,7 +6,9 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) #define DEFINE_SIMPLE_JUMP_TABLE_PROG(NAME, SRC_REG, OFF, IMM, OUTCOME) \ \ @@ -586,6 +588,6 @@ nop_%=: \ : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ +#endif /* gotox: x86, arm64, powerpc, riscv64 */ char _license[] SEC("license") = "GPL"; From 6ce75af4f59eb7065859cba7f6a563dfb1b7ab75 Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Mon, 28 Sep 2026 15:18:44 +0800 Subject: [PATCH 463/660] selftests/bpf: Enable gotox liveness test for riscv64 The riscv64 JIT now supports gotox, so enable the gotox liveness test in compute_live_registers.c on riscv64 as well. The guard is 64-bit only because the RV32 JIT does not implement gotox. Signed-off-by: Chen Pei Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928071845.2432-4-cp0613@linux.alibaba.com --- tools/testing/selftests/bpf/progs/compute_live_registers.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index b0aaf5961ac9..1ff24bbc7beb 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -496,7 +496,8 @@ __naked void subprog_ret_reg_pair(void) ::: __clobber_all); } -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) SEC("socket") __log_level(2) @@ -535,7 +536,7 @@ void gotox(void) : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 */ +#endif /* gotox: x86, arm64, riscv64 */ /* to retain debug info for BTF generation */ void kfunc_root(void) From eec2444d9b9d8cd2bf5e7f54d830325c9b2a1c7d Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Mon, 28 Sep 2026 15:18:45 +0800 Subject: [PATCH 464/660] selftests/bpf: Enable instruction array test for riscv64 As the riscv64 JIT now supports instruction array (gotox jump tables), enable the corresponding bpf selftest for riscv64. The guard is 64-bit only because the RV32 JIT does not implement gotox. Signed-off-by: Chen Pei Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260928071845.2432-5-cp0613@linux.alibaba.com --- tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index bfedeed24ac5..4ffe4fe6b78e 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -4,7 +4,8 @@ #include #include -#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; From cf91a7db92c6fa5294fb97da56e7e532e2ffa4d4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 9 Oct 2026 12:23:18 +0100 Subject: [PATCH 465/660] resolve_btfids: Build separate main and inline BTF objects Extract .BTF.inline via the following logic: - Traverse all types reachable from LOC_PARAM, LOC_PROTO, LOCSEC entries and mark them as having LOC color. - Traverse all types reachable from the remaining types and mark them as having MAIN or SHARED. - Create a new base BTF object and copy all MAIN/SHARED types there. - Create a new .BTF.inline object with base set to the new base BTF object, and copy all LOC types there. This way .BTF.inline can refer to types and strings in the new base, and strings used only by .BTF.inline remain in it. Additionally, all BTF_KIND_FUNC types that map to ELF symbols with the same name are forced to be in the base BTF. Same for types referenced from BTF id sets. Regular types referenced only from .BTF.inline remain in the base BTF, and sizes are 5 and 9.5Mb for vmlinux and vmlinux.inline respectively. Signed-off-by: Eduard Zingerman Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-2-alan.maguire@oracle.com --- tools/bpf/resolve_btfids/Build | 1 + tools/bpf/resolve_btfids/btf_colors.c | 139 +++++++++++++++++++ tools/bpf/resolve_btfids/btf_colors.h | 20 +++ tools/bpf/resolve_btfids/main.c | 193 +++++++++++++++++++++++--- tools/testing/selftests/bpf/Makefile | 4 +- tools/testing/selftests/hid/Makefile | 2 + 6 files changed, 336 insertions(+), 23 deletions(-) create mode 100644 tools/bpf/resolve_btfids/btf_colors.c create mode 100644 tools/bpf/resolve_btfids/btf_colors.h diff --git a/tools/bpf/resolve_btfids/Build b/tools/bpf/resolve_btfids/Build index 077de3829c72..602e1508bf09 100644 --- a/tools/bpf/resolve_btfids/Build +++ b/tools/bpf/resolve_btfids/Build @@ -1,6 +1,7 @@ hostprogs := resolve_btfids resolve_btfids-y += main.o +resolve_btfids-y += btf_colors.o resolve_btfids-y += rbtree.o resolve_btfids-y += zalloc.o resolve_btfids-y += string.o diff --git a/tools/bpf/resolve_btfids/btf_colors.c b/tools/bpf/resolve_btfids/btf_colors.c new file mode 100644 index 000000000000..347ae85d6bc7 --- /dev/null +++ b/tools/bpf/resolve_btfids/btf_colors.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include "btf_colors.h" + +/* + * Functions in this file mainly exist to refer to functions from libbpf_internal.h, + * which can't be included in main.c because of the u32 poison and pr_warn macro conflicts. + */ + +/* + * Marks `root_id` and local types reachable from it with `color`. + * `colors` is indexed by source ID; worklist has room for every local type. + */ +void btf_mark_reachable(struct btf *btf, __u32 root_id, enum btf_color color, + __u8 *colors, __u32 *worklist) +{ + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 pending = 0; + + if (root_id < start_id || (colors[root_id] & color) == color) + return; + colors[root_id] |= color; + worklist[pending++] = root_id; + while (pending) { + const struct btf_type *t = btf__type_by_id(btf, worklist[--pending]); + struct btf_field_iter it; + __u32 *id; + + btf_field_iter_init(&it, (struct btf_type *)t, BTF_FIELD_ITER_IDS); + while ((id = btf_field_iter_next(&it))) { + if (*id < start_id || (colors[*id] & color) == color) + continue; + colors[*id] |= color; + worklist[pending++] = *id; + } + } +} + +static int cmp_loc(const void *a, const void *b) +{ + const struct btf_loc *la = a, *lb = b; + + if (la->func != lb->func) + return la->func < lb->func ? -1 : 1; + if (la->offset != lb->offset) + return la->offset < lb->offset ? -1 : 1; + if (la->loc_proto != lb->loc_proto) + return la->loc_proto < lb->loc_proto ? -1 : 1; + return 0; +} + +static void remap(struct btf *btf, const __u32 *id_map, __u32 src_start_id) +{ + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 i, type_cnt = btf__type_cnt(btf); + + for (i = start_id; i < type_cnt; i++) { + struct btf_type *t = (struct btf_type *)btf__type_by_id(btf, i); + struct btf_field_iter it; + __u32 *id; + + btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + while ((id = btf_field_iter_next(&it))) { + /* Void and types in the original ancestor are unchanged. */ + if (*id >= src_start_id) + *id = id_map[*id - src_start_id]; + } + if (btf_is_locsec(t)) + qsort(btf_locsec_locs(t), btf_vlen(t), sizeof(struct btf_loc), cmp_loc); + } +} + +/* + * Split the `src` into `main_out` base and `inline_out`, + * according to `colors` array. Relative ordering remains + * the same as in `src`. + */ +int btf_split_by_color(struct btf *src, const __u8 *colors, + struct btf **main_out, struct btf **inline_out) +{ + LIBBPF_OPTS(btf_new_opts, opts, + .base_btf = (struct btf *)btf__base_btf(src), + .add_layout = btf_header(src)->layout_len != 0, + ); + struct btf *main_btf, *inline_btf = NULL; + __u32 start_id = opts.base_btf ? btf__type_cnt(opts.base_btf) : 1; + __u32 type_cnt = btf__type_cnt(src); + __u32 *id_map, i; + int err = -ENOMEM; + + *main_out = NULL; + *inline_out = NULL; + id_map = malloc((type_cnt - start_id ?: 1) * sizeof(*id_map)); + if (!id_map) + return -ENOMEM; + main_btf = btf__new_empty_opts(&opts); + if (!main_btf) + goto out; + btf__set_endianness(main_btf, btf__endianness(src)); + /* Copy MAIN and SHARED marked types to `main_out`. */ + for (i = start_id; i < type_cnt; i++) { + if (colors[i] == BTF_COLOR_LOC) + continue; + err = btf__add_type(main_btf, src, btf__type_by_id(src, i)); + if (err < 0) + goto out; + id_map[i - start_id] = err; + } + /* + * Copy LOC marked types to `inline_out`. + * `main_btf` types count is stable at this point. + */ + inline_btf = btf__new_empty_split(main_btf); + if (!inline_btf) { + err = -ENOMEM; + goto out; + } + for (i = start_id; i < type_cnt; i++) { + if (colors[i] != BTF_COLOR_LOC) + continue; + err = btf__add_type(inline_btf, src, btf__type_by_id(src, i)); + if (err < 0) + goto out; + id_map[i - start_id] = err; + } + remap(main_btf, id_map, start_id); + remap(inline_btf, id_map, start_id); + *main_out = main_btf; + *inline_out = inline_btf; + free(id_map); + return 0; +out: + btf__free(inline_btf); + btf__free(main_btf); + free(id_map); + return err; +} diff --git a/tools/bpf/resolve_btfids/btf_colors.h b/tools/bpf/resolve_btfids/btf_colors.h new file mode 100644 index 000000000000..9cea757cb356 --- /dev/null +++ b/tools/bpf/resolve_btfids/btf_colors.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __RESOLVE_BTFIDS_BTF_COLORS_H +#define __RESOLVE_BTFIDS_BTF_COLORS_H + +#include + +enum btf_color { + BTF_COLOR_NONE = 0, + BTF_COLOR_MAIN = 1, + BTF_COLOR_LOC = 2, + BTF_COLOR_SHARED = BTF_COLOR_MAIN | BTF_COLOR_LOC, +}; + +void btf_mark_reachable(struct btf *btf, __u32 root, enum btf_color color, + __u8 *colors, __u32 *worklist); + +int btf_split_by_color(struct btf *src, const __u8 *colors, + struct btf **main_out, struct btf **inline_out); + +#endif diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 37d7e7224207..15e7be13061f 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -89,6 +89,7 @@ #include #include #include +#include "btf_colors.h" #include #define BTF_IDS_SECTION ".BTF_ids" @@ -142,7 +143,10 @@ struct object { struct btf *btf; struct btf *base_btf; + struct btf *inline_btf; + struct btf *orig_btf; bool distill_base; + bool extract_inline; struct { int fd; @@ -154,6 +158,10 @@ struct object { size_t strtabidx; unsigned long idlist_addr; int encoding; + const char **func_symbols; + u32 func_symbols_cnt; + u32 func_symbols_cap; + bool func_symbols_collected; } efile; struct rb_root sets; @@ -572,6 +580,58 @@ static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr) return res ? res->name : NULL; } +static int cmp_func_symbol(const void *a, const void *b) +{ + const char * const *name = a; + const char * const *other = b; + + return strcmp(*name, *other); +} + +static int collect_func_symbols(struct object *obj) +{ + Elf_Scn *scn; + GElf_Shdr sh; + int n, i; + + if (obj->efile.symbols_shndx == -1) + return 0; + + scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx); + if (!scn || gelf_getshdr(scn, &sh) != &sh || !sh.sh_entsize) + return -EINVAL; + n = sh.sh_size / sh.sh_entsize; + + for (i = 0; i < n; i++) { + GElf_Sym sym; + const char *name; + + if (!gelf_getsym(obj->efile.symbols, i, &sym)) + return -EINVAL; + if (GELF_ST_TYPE(sym.st_info) != STT_FUNC || + sym.st_shndx == SHN_UNDEF || !sym.st_name) + continue; + name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name); + if (!name) + return -EINVAL; + if (ensure_mem(&obj->efile.func_symbols, &obj->efile.func_symbols_cap, + obj->efile.func_symbols_cnt + 1)) + return -ENOMEM; + obj->efile.func_symbols[obj->efile.func_symbols_cnt++] = name; + } + + qsort(obj->efile.func_symbols, obj->efile.func_symbols_cnt, + sizeof(*obj->efile.func_symbols), cmp_func_symbol); + obj->efile.func_symbols_collected = true; + return 0; +} + +static bool has_func_symbol(const struct object *obj, const char *name) +{ + return bsearch(&name, obj->efile.func_symbols, obj->efile.func_symbols_cnt, + sizeof(*obj->efile.func_symbols), cmp_func_symbol) != NULL; +} + static int symbols_collect(struct object *obj) { Elf_Scn *scn = NULL; @@ -688,8 +748,9 @@ static int load_btf(struct object *obj) if (err) { pr_err("FAILED: load base BTF from %s: %s\n", obj->base_btf_path, strerror(-err)); - goto out_err; + return err; } + obj->base_btf = base_btf; } btf = btf__parse_split(obj->btf_path ?: obj->path, base_btf); @@ -697,20 +758,12 @@ static int load_btf(struct object *obj) if (err) { pr_err("FAILED: load BTF from %s: %s\n", obj->btf_path ?: obj->path, strerror(-err)); - goto out_err; + return err; } - obj->base_btf = base_btf; obj->btf = btf; return 0; - -out_err: - btf__free(base_btf); - btf__free(btf); - obj->base_btf = NULL; - obj->btf = NULL; - return err; } static int symbols_resolve(struct object *obj) @@ -1510,6 +1563,84 @@ out: return err; } +static bool keep_in_base_btf(struct object *obj, const struct btf_type *t) +{ + const char *name; + + switch (btf_kind(t)) { + case BTF_KIND_FUNC: + name = btf__name_by_offset(obj->btf, t->name_off); + return !obj->efile.func_symbols_collected || has_func_symbol(obj, name) || + btf_id__find(&obj->funcs, name); + case BTF_KIND_FUNC_PROTO: + case BTF_KIND_LOC_PARAM: + case BTF_KIND_LOC_PROTO: + case BTF_KIND_LOCSEC: + return false; + default: + /* + * Ordinary types stay in base btf even if only inline records use them. + * Should we relax this? + */ + return true; + } +} + +/* Mark which types should go to .BTF.inline */ +static int color_btf(struct object *obj, __u8 *colors) +{ + struct btf *btf = obj->btf; + const struct btf *base = btf__base_btf(btf); + __u32 start_id = base ? btf__type_cnt(base) : 1; + __u32 type_cnt = btf__type_cnt(btf), i; + __u32 *worklist; + + worklist = malloc((type_cnt - start_id ?: 1) * sizeof(*worklist)); + if (!worklist) + return -ENOMEM; + /* First, mark all types reachable from LOC BTF entries. */ + for (i = start_id; i < type_cnt; i++) { + const struct btf_type *t = btf__type_by_id(btf, i); + + if (!btf_is_locsec(t) && !btf_is_loc_proto(t) && !btf_is_loc_param(t)) + continue; + btf_mark_reachable(btf, i, BTF_COLOR_LOC, colors, worklist); + } + /* Next, traverse non-marked types converting some LOC markings to SHARED. */ + for (i = start_id; i < type_cnt; i++) { + const struct btf_type *t = btf__type_by_id(btf, i); + + if (colors[i] == BTF_COLOR_LOC && !keep_in_base_btf(obj, t)) + continue; + btf_mark_reachable(btf, i, BTF_COLOR_MAIN, colors, worklist); + } + free(worklist); + return 0; +} + +static int extract_inline_btf(struct object *obj) +{ + struct btf *btf = obj->btf, *main_btf, *inline_btf; + __u8 *colors; + int err; + + colors = calloc(btf__type_cnt(btf), sizeof(*colors)); + if (!colors) + return -ENOMEM; + err = color_btf(obj, colors); + if (err) + goto out; + err = btf_split_by_color(btf, colors, &main_btf, &inline_btf); + if (!err) { + obj->orig_btf = btf; + obj->btf = main_btf; + obj->inline_btf = inline_btf; + } +out: + free(colors); + return err; +} + /* * Sort types by name in ascending order resulting in all * anonymous types being placed before named types. @@ -1583,14 +1714,14 @@ static int finalize_btf(struct object *obj) err = btf__dedup(obj->btf, NULL); if (err) { pr_err("FAILED to dedup BTF: %s\n", strerror(errno)); - goto out_err; + return err; } if (obj->base_btf && obj->distill_base) { err = btf__distill_base(obj->btf, &base_btf, &btf); if (err) { pr_err("FAILED to distill base BTF: %s\n", strerror(errno)); - goto out_err; + return err; } btf__free(obj->base_btf); @@ -1602,18 +1733,18 @@ static int finalize_btf(struct object *obj) err = sort_btf_by_name(obj->btf); if (err) { pr_err("FAILED to sort BTF: %s\n", strerror(errno)); - goto out_err; + return err; + } + + if (obj->extract_inline) { + err = extract_inline_btf(obj); + if (err) { + pr_err("FAILED to extract inline BTF: %s\n", strerror(-err)); + return err; + } } return 0; - -out_err: - btf__free(base_btf); - btf__free(btf); - obj->base_btf = NULL; - obj->btf = NULL; - - return err; } static inline int make_out_path(char *buf, u32 buf_sz, const char *in_path, const char *suffix) @@ -1773,6 +1904,8 @@ int main(int argc, const char **argv) "turn warnings into errors"), OPT_BOOLEAN(0, "distill_base", &obj.distill_base, "distill --btf_base and emit .BTF.base section data"), + OPT_BOOLEAN(0, "inline", &obj.extract_inline, + "extract location BTF into a .BTF.inline file"), OPT_STRING(0, "patch_btfids", &btfids_path, "file", "path to .BTF_ids section data blob to patch into ELF file"), OPT_END() @@ -1791,6 +1924,8 @@ int main(int argc, const char **argv) if (elf_collect(&obj)) goto out; + if (obj.extract_inline && collect_func_symbols(&obj)) + goto out; /* * We did not find .BTF_ids section or symbols section, @@ -1841,18 +1976,32 @@ dump_btf: if (err) goto out; } + if (obj.inline_btf) { + err = make_out_path(out_path, sizeof(out_path), obj.path, BTF_ELF_SEC ".inline"); + err = err ?: dump_raw_btf(obj.inline_btf, out_path); + if (err) + goto out; + } if (!(fatal_warnings && warnings)) err = 0; out: - btf__free(obj.base_btf); + /* + * Free BTFs in reverse dependency order. inline_btf borrows obj.btf; + * obj.btf can borrow a base owned by orig_btf or obj.base_btf, and + * orig_btf can also borrow obj.base_btf. + */ + btf__free(obj.inline_btf); btf__free(obj.btf); + btf__free(obj.orig_btf); + btf__free(obj.base_btf); btf_id__free_all(&obj.structs); btf_id__free_all(&obj.unions); btf_id__free_all(&obj.typedefs); btf_id__free_all(&obj.funcs); btf_id__free_all(&obj.sets); free(obj.addr_syms); + free(obj.efile.func_symbols); if (obj.efile.elf) { elf_end(obj.efile.elf); close(obj.efile.fd); diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index a22be7efd1fa..80d8bbbc3084 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -343,7 +343,9 @@ else endif $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ - $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \ $(TOOLSDIR)/lib/rbtree.c \ $(TOOLSDIR)/lib/zalloc.c \ $(TOOLSDIR)/lib/string.c \ diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 2f423de83147..d1b65fd37911 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile @@ -148,6 +148,8 @@ endif $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \ $(TOOLSDIR)/lib/rbtree.c \ $(TOOLSDIR)/lib/zalloc.c \ $(TOOLSDIR)/lib/string.c \ From aac67645215d5a38cff9f86110b81c220e5e8182 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:19 +0100 Subject: [PATCH 466/660] resolve_btfids: Populate BTF link section Allow the final BTF patch pass to populate one or more BTF link records. A link identifies the BTF section to be delivered, its carrier module, and the raw finalized BTF payload: --btf_link
:: Use the raw file size and SHA-256 digest to populate the corresponding
.link record. This avoids reparsing BTF during final ELF patching and supports inline vmlinux BTF linking. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-3-alan.maguire@oracle.com --- tools/bpf/resolve_btfids/btf_colors.c | 6 + tools/bpf/resolve_btfids/btf_colors.h | 4 + tools/bpf/resolve_btfids/main.c | 234 +++++++++++++++++++++++++- 3 files changed, 241 insertions(+), 3 deletions(-) diff --git a/tools/bpf/resolve_btfids/btf_colors.c b/tools/bpf/resolve_btfids/btf_colors.c index 347ae85d6bc7..5d307d70e7f2 100644 --- a/tools/bpf/resolve_btfids/btf_colors.c +++ b/tools/bpf/resolve_btfids/btf_colors.c @@ -7,6 +7,12 @@ * which can't be included in main.c because of the u32 poison and pr_warn macro conflicts. */ +void btf_sha256(const void *data, size_t len, + __u8 out[BTF_SHA256_DIGEST_LENGTH]) +{ + libbpf_sha256(data, len, out); +} + /* * Marks `root_id` and local types reachable from it with `color`. * `colors` is indexed by source ID; worklist has room for every local type. diff --git a/tools/bpf/resolve_btfids/btf_colors.h b/tools/bpf/resolve_btfids/btf_colors.h index 9cea757cb356..56a85a242d6b 100644 --- a/tools/bpf/resolve_btfids/btf_colors.h +++ b/tools/bpf/resolve_btfids/btf_colors.h @@ -11,6 +11,10 @@ enum btf_color { BTF_COLOR_SHARED = BTF_COLOR_MAIN | BTF_COLOR_LOC, }; +#define BTF_SHA256_DIGEST_LENGTH 32 + +void btf_sha256(const void *data, size_t len, + __u8 out[BTF_SHA256_DIGEST_LENGTH]); void btf_mark_reachable(struct btf *btf, __u32 root, enum btf_color color, __u8 *colors, __u32 *worklist); diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 15e7be13061f..f737fac60fa2 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -93,6 +93,7 @@ #include #define BTF_IDS_SECTION ".BTF_ids" +#define BTF_LINK_MODULE_NAME_MAX 64 #define BTF_ID_PREFIX "__BTF_ID__" #define BTF_STRUCT "struct" @@ -158,6 +159,7 @@ struct object { size_t strtabidx; unsigned long idlist_addr; int encoding; + int elf_class; const char **func_symbols; u32 func_symbols_cnt; u32 func_symbols_cap; @@ -180,6 +182,19 @@ struct object { u32 addr_syms_cap; }; +struct btf_link { + char *value; + char *section; + char *module; + char *btf_path; +}; + +struct btf_links { + struct btf_link *links; + u32 cnt; + u32 cap; +}; + #define DECL_TAG_FASTCALL "bpf_fastcall" #define DECL_TAG_KFUNC "bpf_kfunc" @@ -267,6 +282,49 @@ static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz) #define ensure_mem(arr_ptr, cap_ptr, cnt) \ __ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr))) +static int parse_btf_link(const struct option *opt, const char *arg, int unset) +{ + struct btf_links *links = opt->value; + struct btf_link *link; + char *separator; + + if (unset) + return -EINVAL; + if (__ensure_mem((void **)&links->links, &links->cap, links->cnt + 1, + sizeof(*links->links))) + return -ENOMEM; + link = &links->links[links->cnt]; + memset(link, 0, sizeof(*link)); + link->value = strdup(arg); + if (!link->value) + return -ENOMEM; + link->section = link->value; + separator = strchr(link->section, ':'); + if (!separator || separator == link->section) + goto err_value; + *separator++ = '\0'; + link->module = separator; + separator = strchr(link->module, ':'); + if (!separator || separator == link->module || !separator[1]) + goto err_value; + *separator++ = '\0'; + link->btf_path = separator; + links->cnt++; + return 0; +err_value: + free(link->value); + return -EINVAL; +} + +static void free_btf_links(struct btf_links *links) +{ + u32 i; + + for (i = 0; i < links->cnt; i++) + free(links->links[i].value); + free(links->links); +} + static bool is_btf_id(const char *name) { return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1); @@ -488,6 +546,7 @@ static int elf_collect(struct object *obj) return -1; } obj->efile.encoding = ehdr.e_ident[EI_DATA]; + obj->efile.elf_class = ehdr.e_ident[EI_CLASS]; /* * Scan all the elf sections and look for save data @@ -1030,6 +1089,153 @@ static int dump_raw_btf(struct btf *btf, const char *out_path) return 0; } +static int patch_btf_link(struct object *obj, const struct btf_link *link) +{ + char section[128]; + void *raw_btf_data = NULL; + size_t shdrstrndx, module_name_len; + int fd = -1, len, err = -1; + FILE *btf_file = NULL; + Elf_Scn *scn = NULL; + u32 raw_btf_size; + struct stat st; + Elf_Data *data; + GElf_Shdr sh; + char *name; + Elf *elf = NULL; + u8 *size; + + if (stat(link->btf_path, &st)) { + err = -errno; + pr_err("FAILED to stat BTF file %s: %s\n", + link->btf_path, strerror(errno)); + return err; + } + if (!S_ISREG(st.st_mode) || st.st_size <= 0 || st.st_size > UINT_MAX) { + pr_err("FAILED: invalid BTF file size or type: %s\n", link->btf_path); + return -EINVAL; + } + raw_btf_size = st.st_size; + raw_btf_data = malloc(raw_btf_size); + if (!raw_btf_data) { + pr_err("FAILED to allocate %u bytes for BTF file %s\n", + raw_btf_size, link->btf_path); + return -ENOMEM; + } + btf_file = fopen(link->btf_path, "rb"); + if (!btf_file) { + err = -errno; + pr_err("FAILED to open BTF file %s: %s\n", + link->btf_path, strerror(errno)); + goto out; + } + if (fread(raw_btf_data, raw_btf_size, 1, btf_file) != 1) { + err = -EIO; + pr_err("FAILED to read BTF file %s\n", link->btf_path); + goto out; + } + fclose(btf_file); + btf_file = NULL; + if (obj->efile.elf_class == ELFCLASS32) { + module_name_len = BTF_LINK_MODULE_NAME_MAX - 4; + } else if (obj->efile.elf_class == ELFCLASS64) { + module_name_len = BTF_LINK_MODULE_NAME_MAX - 8; + } else { + err = -EINVAL; + pr_err("FAILED: unsupported ELF class %u in %s\n", + obj->efile.elf_class, obj->path); + goto out; + } + if (strlen(link->module) >= module_name_len) { + err = -EINVAL; + pr_err("FAILED: module name too long for BTF link: %s\n", link->module); + goto out; + } + len = snprintf(section, sizeof(section), "%s.link", link->section); + if (len < 0 || len >= sizeof(section)) { + err = -ENAMETOOLONG; + pr_err("FAILED: BTF link section name too long: %s.link\n", link->section); + goto out; + } + + fd = open(obj->path, O_RDWR); + if (fd < 0) { + err = -errno; + pr_err("FAILED to open %s: %s\n", obj->path, strerror(errno)); + goto out; + } + elf = elf_begin(fd, ELF_C_RDWR_MMAP, NULL); + if (!elf) { + err = -EINVAL; + pr_err("FAILED to create ELF descriptor for %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + elf_flagelf(elf, ELF_C_SET, ELF_F_LAYOUT); + if (elf_getshdrstrndx(elf, &shdrstrndx)) { + pr_err("FAILED to get section name index from %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + while ((scn = elf_nextscn(elf, scn))) { + if (gelf_getshdr(scn, &sh) != &sh) { + pr_err("FAILED to get section header from %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + name = elf_strptr(elf, shdrstrndx, sh.sh_name); + if (name && !strcmp(name, section)) + break; + } + if (!scn) { + pr_err("FAILED: section %s not found in %s\n", section, obj->path); + goto out; + } + data = elf_getdata(scn, NULL); + if (sh.sh_type != SHT_PROGBITS || !data || !data->d_buf || + data->d_size != module_name_len + + BTF_SHA256_DIGEST_LENGTH + sizeof(u32)) { + pr_err("FAILED: section %s is not a writable BTF link\n", section); + goto out; + } + memset(data->d_buf, 0, data->d_size); + memcpy(data->d_buf, link->module, strlen(link->module) + 1); + btf_sha256(raw_btf_data, raw_btf_size, + (u8 *)data->d_buf + module_name_len); + size = (u8 *)data->d_buf + module_name_len + BTF_SHA256_DIGEST_LENGTH; + if (obj->efile.encoding == ELFDATA2LSB) { + size[0] = raw_btf_size; + size[1] = raw_btf_size >> 8; + size[2] = raw_btf_size >> 16; + size[3] = raw_btf_size >> 24; + } else if (obj->efile.encoding == ELFDATA2MSB) { + size[0] = raw_btf_size >> 24; + size[1] = raw_btf_size >> 16; + size[2] = raw_btf_size >> 8; + size[3] = raw_btf_size; + } else { + pr_err("FAILED: unsupported ELF data encoding %u in %s\n", + obj->efile.encoding, obj->path); + goto out; + } + elf_flagdata(data, ELF_C_SET, ELF_F_DIRTY); + if (elf_update(elf, ELF_C_WRITE) < 0) { + pr_err("FAILED to update ELF file %s: %s\n", + obj->path, elf_errmsg(-1)); + goto out; + } + err = 0; +out: + if (elf) + elf_end(elf); + if (fd >= 0) + close(fd); + if (btf_file) + fclose(btf_file); + free(raw_btf_data); + return err; +} + static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s32 type_id) { const struct btf_type *t = btf__type_by_id(btf, type_id); @@ -1871,7 +2077,7 @@ out: static const char * const resolve_btfids_usage[] = { "resolve_btfids [] ", - "resolve_btfids --patch_btfids <.BTF_ids file> ", + "resolve_btfids --patch_btfids <.BTF_ids file> [--btf_link ] ", NULL }; @@ -1889,6 +2095,7 @@ int main(int argc, const char **argv) .sets = RB_ROOT, }; const char *btfids_path = NULL; + struct btf_links btf_links = {}; bool fatal_warnings = false; bool resolve_btfids = true; char out_path[PATH_MAX]; @@ -1906,6 +2113,8 @@ int main(int argc, const char **argv) "distill --btf_base and emit .BTF.base section data"), OPT_BOOLEAN(0, "inline", &obj.extract_inline, "extract location BTF into a .BTF.inline file"), + OPT_CALLBACK(0, "btf_link", &btf_links, "section:module:btf-file", + "patch a BTF link with --patch_btfids", parse_btf_link), OPT_STRING(0, "patch_btfids", &btfids_path, "file", "path to .BTF_ids section data blob to patch into ELF file"), OPT_END() @@ -1916,12 +2125,30 @@ int main(int argc, const char **argv) PARSE_OPT_STOP_AT_NON_OPTION); if (argc != 1) usage_with_options(resolve_btfids_usage, btfid_options); + if (btf_links.cnt && !btfids_path) { + pr_err("--btf_link requires --patch_btfids\n"); + goto out; + } obj.path = argv[0]; - if (btfids_path) - return patch_btfids(btfids_path, obj.path); + if (btfids_path) { + err = patch_btfids(btfids_path, obj.path); + if (err || !btf_links.cnt) + goto out; + if (elf_collect(&obj)) { + err = -EINVAL; + goto out; + } + for (u32 i = 0; i < btf_links.cnt; i++) { + struct btf_link *link = &btf_links.links[i]; + err = patch_btf_link(&obj, link); + if (err) + break; + } + goto out; + } if (elf_collect(&obj)) goto out; if (obj.extract_inline && collect_func_symbols(&obj)) @@ -1986,6 +2213,7 @@ dump_btf: if (!(fatal_warnings && warnings)) err = 0; out: + free_btf_links(&btf_links); /* * Free BTFs in reverse dependency order. inline_btf borrows obj.btf; * obj.btf can borrow a base owned by orig_btf or obj.base_btf, and From 672981f4aec1d0131761aec60298312464bd7a50 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:20 +0100 Subject: [PATCH 467/660] kbuild, btf: Expose vmlinux inline BTF through sysfs Expose BTF inline-location information from vmlinux .BTF.inline sections in /sys/kernel/btf as vmlinux.inline. Embedded vmlinux inline BTF supports read-only mmap(). Provide runtime support for a module-delivered vmlinux inline BTF carrier. Its link record binds the carrier module name, BTF size, and SHA-256 digest to the running kernel before its data is made available through a lazy vmlinux.inline sysfs attribute. The carrier is handled before ordinary module BTF registration, so its incidental .BTF data is not exposed. Make CONFIG_DEBUG_INFO_BTF_INLINE tristate. When configured as a module, retain vmlinux final split inline BTF for btf_vmlinux_inline.ko instead of embedding it in vmlinux. Reserve .BTF.inline.link in vmlinux and extend the final resolve_btfids patch pass to bind btf_vmlinux_inline to the raw vmlinux.BTF.inline payload. This preserves the normal built-in configuration while allowing the large inline BTF payload to load on demand. Normal module inline BTF is handled separately, after its references have been relocated against the final module BTF. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-4-alan.maguire@oracle.com --- .gitignore | 1 + Makefile | 1 + include/asm-generic/vmlinux.lds.h | 31 ++++++ include/linux/btf.h | 37 +++++++ include/linux/module.h | 4 + kernel/bpf/Makefile | 1 + kernel/bpf/btf.c | 160 ++++++++++++++++++++++++------ kernel/bpf/btf_vmlinux_inline.c | 29 ++++++ kernel/bpf/sysfs_btf.c | 149 ++++++++++++++++++++++++++-- kernel/module/main.c | 7 ++ lib/Kconfig.debug | 18 ++++ scripts/Makefile.btf | 7 ++ scripts/gen-btf.sh | 34 ++++++- scripts/link-vmlinux.sh | 8 +- scripts/package/kernel.spec | 10 +- 15 files changed, 446 insertions(+), 51 deletions(-) create mode 100644 kernel/bpf/btf_vmlinux_inline.c diff --git a/.gitignore b/.gitignore index 9875120ea7bd..d54bdaac9981 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ modules.order /vmlinux.symvers /vmlinux.thinlto-index /vmlinux.unstripped +/vmlinux.BTF.inline /vmlinux-gdb.py /vmlinuz /System.map diff --git a/Makefile b/Makefile index 6b8b812f23b9..b3053dd3c105 100644 --- a/Makefile +++ b/Makefile @@ -1769,6 +1769,7 @@ endif # CONFIG_MODULES CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ + vmlinux.BTF.inline \ vmlinux.thinlto-index builtin.order \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b2988aa12f66..b17251bc63f5 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -675,12 +675,43 @@ /* * .BTF */ +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE +#define BTF_INLINE \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline : AT(ADDR(.BTF.inline) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.BTF.inline, _BTF_inline) \ + } +#else +#define BTF_INLINE +#endif + +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE_MODULE +/* __MODULE_NAME_LEN + SHA-256 digest + u32 BTF size */ +#ifdef CONFIG_64BIT +#define BTF_LINK_SIZE (64 - 8 + 32 + 4) +#else +#define BTF_LINK_SIZE (64 - 4 + 32 + 4) +#endif +#define BTF_INLINE_LINK \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline.link : AT(ADDR(.BTF.inline.link) - LOAD_OFFSET) { \ + __start_BTF_inline_link = .; \ + BYTE(0) \ + . += BTF_LINK_SIZE - 1; \ + __stop_BTF_inline_link = .; \ + } +#else +#define BTF_INLINE_LINK +#endif + #ifdef CONFIG_DEBUG_INFO_BTF #define BTF \ . = ALIGN(PAGE_SIZE); \ .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ BOUNDED_SECTION_BY(.BTF, _BTF) \ } \ + BTF_INLINE \ + BTF_INLINE_LINK \ . = ALIGN(PAGE_SIZE); \ .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ *(.BTF_ids) \ diff --git a/include/linux/btf.h b/include/linux/btf.h index 4b63bb91550a..f731c7a5f14c 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include +#include #include #include #include @@ -118,6 +119,19 @@ struct btf_show; struct btf_id_set; struct bpf_prog; +#define BTF_LINK_SHA256_LEN 32 + +/* + * A .BTF.link or .BTF.inline.link section identifies the module which + * carries a vmlinux BTF section. The module name is NUL terminated and the + * field is padded to __MODULE_NAME_LEN. + */ +struct btf_link { + char module_name[__MODULE_NAME_LEN]; + u8 sha256[BTF_LINK_SHA256_LEN]; + u32 btf_size; +} __packed; + typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { @@ -624,6 +638,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx); +#if IS_ENABLED(CONFIG_SYSFS) +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name); +bool sysfs_btf_update(struct bin_attribute *attr, void *data); +void sysfs_btf_remove(struct bin_attribute *attr); +#else +static inline struct bin_attribute * +sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap, + const char *lazy_module_name) +{ + return NULL; +} + +static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return false; +} + +static inline void sysfs_btf_remove(struct bin_attribute *attr) +{ +} +#endif + static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) { if (!btf_type_is_ptr(t)) diff --git a/include/linux/module.h b/include/linux/module.h index 96cc98568eea..a6abde12a458 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -503,6 +503,10 @@ struct module { void *btf_data; void *btf_base_data; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + unsigned int btf_inline_data_size; + void *btf_inline_data; +#endif #ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index d61cbb59ad08..269810b128c4 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -42,6 +42,7 @@ obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o endif ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF_INLINE) += btf_vmlinux_inline.o endif ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index f4d119f3d5ef..51992cfc81c3 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -25,11 +25,12 @@ #include #include #include -#include #include -#include #include #include +#include +#include +#include #include @@ -9126,21 +9127,132 @@ static DEFINE_MUTEX(btf_module_mutex); static void purge_cand_cache(struct btf *btf); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline_link[]; +extern char __stop_BTF_inline_link[]; + +static struct bin_attribute *vmlinux_inline_attr; +static const struct btf_link *vmlinux_inline_link; + +static const struct btf_link *btf_parse_link(void *start, unsigned int size) +{ + const struct btf_link *link = start; + unsigned int module_name_len; + + if (!link || size != sizeof(*link) || link->btf_size == 0) + return NULL; + + module_name_len = strnlen(link->module_name, sizeof(link->module_name)); + if (!module_name_len || module_name_len == sizeof(link->module_name) || + memchr_inv(link->module_name + module_name_len + 1, 0, + sizeof(link->module_name) - module_name_len - 1)) + return NULL; + return link; +} + +static bool btf_verify_link(const struct btf_link *link, const struct module *mod, + const void *data, size_t data_size) +{ + u8 actual_digest[BTF_LINK_SHA256_LEN]; + + if (!link || data_size != link->btf_size) { + pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name); + return false; + } + sha256(data, data_size, actual_digest); + if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) { + pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name); + return false; + } + + return true; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ + vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link, + __stop_BTF_inline_link - + __start_BTF_inline_link); + if (!vmlinux_inline_link) { + pr_warn("invalid vmlinux inline BTF link\n"); + return; + } + vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL, + vmlinux_inline_link->btf_size, true, + vmlinux_inline_link->module_name); + if (IS_ERR(vmlinux_inline_attr)) { + pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n", + PTR_ERR(vmlinux_inline_attr)); + vmlinux_inline_attr = NULL; + } +} + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name); +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + void *data; + + if (!mod->btf_inline_data || + mod->btf_inline_data_size != vmlinux_inline_link->btf_size) + return -EINVAL; + + data = vmalloc_user(mod->btf_inline_data_size); + if (!data) + return -ENOMEM; + memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size); + + if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) || + !sysfs_btf_update(vmlinux_inline_attr, data)) { + kvfree(data); + return -EINVAL; + } + return 0; +} + +#else + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return false; +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + return 0; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ +} + +#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */ + static int btf_module_notify(struct notifier_block *nb, unsigned long op, void *module) { struct btf_module *btf_mod, *tmp; struct module *mod = module; + struct bin_attribute *attr; struct btf *btf; int err = 0; - if (mod->btf_data_size == 0 || - (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && - op != MODULE_STATE_GOING)) + if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && + op != MODULE_STATE_GOING) goto out; switch (op) { case MODULE_STATE_COMING: + if (btf_module_is_vmlinux_inline(mod)) { + err = btf_module_handle_vmlinux_inline(mod); + break; + } + + if (!mod->btf_data_size) + break; btf_mod = kzalloc_obj(*btf_mod); if (!btf_mod) { err = -ENOMEM; @@ -9173,34 +9285,16 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, list_add(&btf_mod->list, &btf_modules); mutex_unlock(&btf_module_mutex); - if (IS_ENABLED(CONFIG_SYSFS)) { - struct bin_attribute *attr; - - attr = kzalloc_obj(*attr); - if (!attr) - goto out; - - sysfs_bin_attr_init(attr); - attr->attr.name = btf->name; - attr->attr.mode = 0444; - attr->size = btf->data_size; - attr->private = btf->data; - attr->read = sysfs_bin_attr_simple_read; - - err = sysfs_create_bin_file(btf_kobj, attr); - if (err) { - pr_warn("failed to register module [%s] BTF in sysfs: %d\n", - mod->name, err); - kfree(attr); - err = 0; - goto out; - } - - btf_mod->sysfs_attr = attr; + attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL); + if (IS_ERR(attr)) { + err = 0; + goto out; } - + btf_mod->sysfs_attr = attr; break; case MODULE_STATE_LIVE: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -9212,6 +9306,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_unlock(&btf_module_mutex); break; case MODULE_STATE_GOING: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -9226,10 +9322,9 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) - sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); + sysfs_btf_remove(btf_mod->sysfs_attr); purge_cand_cache(btf_mod->btf); btf_put(btf_mod->btf); - kfree(btf_mod->sysfs_attr); kfree(btf_mod); break; } @@ -9246,6 +9341,7 @@ static struct notifier_block btf_module_nb = { static int __init btf_module_init(void) { + btf_vmlinux_inline_lazy_init(); register_module_notifier(&btf_module_nb); return 0; } diff --git a/kernel/bpf/btf_vmlinux_inline.c b/kernel/bpf/btf_vmlinux_inline.c new file mode 100644 index 000000000000..d5c599d9f729 --- /dev/null +++ b/kernel/bpf/btf_vmlinux_inline.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +/* + * Provide kernel BTF inline function information for use by BPF tools. + */ +#include +#include +#include +#include + +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline[]; +extern char __stop_BTF_inline[]; +#endif + +static int __init btf_vmlinux_inline_init(void) +{ +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) + size_t data_size = __stop_BTF_inline - __start_BTF_inline; + + if (data_size) + sysfs_btf_add("vmlinux.inline", __start_BTF_inline, data_size, true, NULL); +#endif + return 0; +} +subsys_initcall(btf_vmlinux_inline_init); + +MODULE_DESCRIPTION("BTF inline information for vmlinux"); +MODULE_LICENSE("GPL"); diff --git a/kernel/bpf/sysfs_btf.c b/kernel/bpf/sysfs_btf.c index 9cbe15ce3540..556fe92c3649 100644 --- a/kernel/bpf/sysfs_btf.c +++ b/kernel/bpf/sysfs_btf.c @@ -6,8 +6,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -15,16 +17,12 @@ extern char __start_BTF[]; extern char __stop_BTF[]; -static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, - const struct bin_attribute *attr, - struct vm_area_struct *vma) +static int btf_sysfs_mmap_check(void *data, size_t size, struct vm_area_struct *vma) { - unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT; + unsigned long pages = PAGE_ALIGN(size) >> PAGE_SHIFT; size_t vm_size = vma->vm_end - vma->vm_start; - phys_addr_t addr = __pa_symbol(__start_BTF); - unsigned long pfn = addr >> PAGE_SHIFT; - if (attr->private != __start_BTF || !PAGE_ALIGNED(addr)) + if (!data || !PAGE_ALIGNED((unsigned long)data)) return -EINVAL; if (vma->vm_pgoff) @@ -33,24 +31,153 @@ static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE)) return -EACCES; - if (pfn + pages < pfn) - return -EINVAL; - if ((vm_size >> PAGE_SHIFT) > pages) return -EINVAL; vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE); + return 0; +} + +static int btf_sysfs_mmap_image(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = READ_ONCE(attr->private); + phys_addr_t addr; + unsigned long pfn; + size_t vm_size = vma->vm_end - vma->vm_start; + int err; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + /* Kernel image addresses can lie inside the vmalloc range on arm64. */ + addr = __pa_symbol(data); + pfn = addr >> PAGE_SHIFT; + if (pfn + (PAGE_ALIGN(attr->size) >> PAGE_SHIFT) < pfn) + return -EINVAL; + return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); } static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { .attr = { .name = "vmlinux", .mode = 0444, }, .read = sysfs_bin_attr_simple_read, - .mmap = btf_sysfs_vmlinux_mmap, + .mmap = btf_sysfs_mmap_image, }; struct kobject *btf_kobj; +struct btf_sysfs_entry { + struct bin_attribute attr; + char *module_name; +}; + +static void *btf_sysfs_lazy_data(const struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + void *data; + + data = smp_load_acquire(&attr->private); + if (!data) { + request_module("%s", entry->module_name); + data = smp_load_acquire(&attr->private); + } + return data; +} + +static ssize_t btf_sysfs_read_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + void *data = btf_sysfs_lazy_data(attr); + + if (!data) + return -ENODEV; + + return memory_read_from_buffer(buf, count, &off, data, attr->size); +} + +static int btf_sysfs_mmap_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = btf_sysfs_lazy_data(attr); + int err; + + if (!data) + return -ENODEV; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + return remap_vmalloc_range(vma, data, 0); +} + +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name) +{ + struct btf_sysfs_entry *entry; + struct bin_attribute *attr; + int err; + + entry = kzalloc_obj(*entry); + if (!entry) + return ERR_PTR(-ENOMEM); + + attr = &entry->attr; + sysfs_bin_attr_init(attr); + attr->attr.mode = 0444; + attr->size = data_size; + attr->private = data; + attr->read = lazy_module_name ? btf_sysfs_read_lazy : sysfs_bin_attr_simple_read; + if (mmap) + attr->mmap = lazy_module_name ? btf_sysfs_mmap_lazy : btf_sysfs_mmap_image; + attr->attr.name = kstrdup(name, GFP_KERNEL); + if (!attr->attr.name) { + err = -ENOMEM; + goto err_free; + } + if (lazy_module_name) { + entry->module_name = kstrdup(lazy_module_name, GFP_KERNEL); + if (!entry->module_name) { + err = -ENOMEM; + goto err_free; + } + } + + err = sysfs_create_bin_file(btf_kobj, attr); + if (err) { + pr_warn("failed to register [%s] BTF in sysfs: %d\n", name, err); + goto err_free; + } + + return attr; + +err_free: + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); + return ERR_PTR(err); +} + +bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return attr && !cmpxchg(&attr->private, NULL, data); +} + +void sysfs_btf_remove(struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + + sysfs_remove_bin_file(btf_kobj, attr); + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); +} + static int __init btf_vmlinux_init(void) { bin_attr_btf_vmlinux.private = __start_BTF; diff --git a/kernel/module/main.c b/kernel/module/main.c index c1b34dc1e89a..5b9eb56164b2 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2723,6 +2723,10 @@ static int find_module_sections(struct module *mod, struct load_info *info) mod->btf_base_data = any_section_objs(info, ".BTF.base", 1, &mod->btf_base_data_size); #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = any_section_objs(info, ".BTF.inline", 1, + &mod->btf_inline_data_size); +#endif #ifdef CONFIG_JUMP_LABEL mod->jump_entries = section_objs(info, "__jump_table", sizeof(*mod->jump_entries), @@ -3176,6 +3180,9 @@ static noinline int do_init_module(struct module *mod) /* .BTF is not SHF_ALLOC and will get removed, so sanitize pointers */ mod->btf_data = NULL; mod->btf_base_data = NULL; +#endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = NULL; #endif /* * We want to free module_init, but be aware that kallsyms may be diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 134b15a44625..2f34619905b6 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -425,6 +425,12 @@ config PAHOLE_HAS_LANG_EXCLUDE otherwise it would emit malformed kernel and module binaries when using DEBUG_INFO_BTF_MODULES. +config PAHOLE_HAS_INLINE + def_bool PAHOLE_VERSION >= 132 + help + Support for the "inline" BTF feature is available. It encodes + information about inline sites and how to retrieve their parameters. + config DEBUG_INFO_BTF_MODULES bool "Generate BTF type information for kernel modules" default y @@ -432,6 +438,18 @@ config DEBUG_INFO_BTF_MODULES help Generate compact split BTF type information for kernel modules. +config DEBUG_INFO_BTF_INLINE + tristate "Provide information about inline sites in BTF" + default n + depends on DEBUG_INFO_BTF && PAHOLE_HAS_INLINE && SYSFS + select DEBUG_INFO_BTF_MODULES if MODULES + help + Generate information about inline sites in .BTF.inline sections. + These sections contain split BTF relative to the kernel or module BTF + and are made available in /sys/kernel/btf with a ".inline" suffix. + The information describes inline locations and how to retrieve their + associated parameters. + config MODULE_ALLOW_BTF_MISMATCH bool "Allow loading modules with non-matching BTF type info" depends on DEBUG_INFO_BTF_MODULES diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index a1812985a61a..98012344a81e 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf @@ -22,7 +22,14 @@ endif pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE) += --lang_exclude=rust +btf-inline := $(CONFIG_DEBUG_INFO_BTF_INLINE) +ifneq ($(btf-inline),) +btf-inline-feat := inline +pahole-flags-$(call test-ge, $(pahole-ver), 132) += --btf_features=$(btf-inline-feat) +endif + export PAHOLE_FLAGS := $(pahole-flags-y) +export BTF_INLINE := $(btf-inline) resolve-btfids-flags-y := resolve-btfids-flags-$(CONFIG_WERROR) += --fatal_warnings diff --git a/scripts/gen-btf.sh b/scripts/gen-btf.sh index 8ca96eb10a69..e638291753d0 100755 --- a/scripts/gen-btf.sh +++ b/scripts/gen-btf.sh @@ -68,6 +68,10 @@ esac gen_btf_data() { + inline="" + if [ -n "${BTF_INLINE}" ] && [ -z "${BTF_BASE}" ]; then + inline="--inline" + fi btf1="${ELF_FILE}.BTF.1" ${PAHOLE} -J ${PAHOLE_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ @@ -76,6 +80,7 @@ gen_btf_data() ${RESOLVE_BTFIDS} ${RESOLVE_BTFIDS_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ + ${inline} \ --btf ${btf1} "${ELF_FILE}" } @@ -83,14 +88,28 @@ gen_btf_o() { btf_data=${ELF_FILE}.btf.o - # Create ${btf_data} which contains just .BTF section but no symbols. Add + # Create ${btf_data} which contains just BTF sections but no symbols. Add # SHF_ALLOC because .BTF will be part of the vmlinux image. --strip-all # deletes all symbols including __start_BTF and __stop_BTF, which will # be redefined in the linker script. echo "" | ${CC} ${CLANG_FLAGS} ${KBUILD_CPPFLAGS} ${KBUILD_CFLAGS} -fno-lto -c -x c -o ${btf_data} - ${OBJCOPY} --add-section .BTF=${ELF_FILE}.BTF \ --set-section-flags .BTF=alloc,readonly ${btf_data} - ${OBJCOPY} --only-section=.BTF --strip-all ${btf_data} + ONLY_SEC="--only-section=.BTF" + btf_inline=${ELF_FILE}.BTF.inline + if [ "${BTF_INLINE}" = "m" ]; then + if [ -f "${btf_inline}" ]; then + cp "${btf_inline}" "${objtree}/vmlinux.BTF.inline" + else + rm -f "${objtree}/vmlinux.BTF.inline" + fi + fi + if [ "${BTF_INLINE}" = "y" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} \ + --set-section-flags .BTF.inline=alloc,readonly ${btf_data} + ONLY_SEC="${ONLY_SEC} --only-section=.BTF.inline" + fi + ${OBJCOPY} ${ONLY_SEC} --strip-all ${btf_data} # Change e_type to ET_REL so that it can be used to link final vmlinux. # GNU ld 2.35+ and lld do not allow an ET_EXEC input. @@ -111,6 +130,15 @@ embed_btf_data() if [ -f "${btf_base}" ]; then ${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE} fi + btf_inline="" + case "${ELF_FILE}" in + */btf_vmlinux_inline.ko) + btf_inline=${BTF_BASE}.BTF.inline + ;; + esac + if [ -n "${BTF_INLINE}" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} ${ELF_FILE} + fi btf_ids="${ELF_FILE}.BTF_ids" if [ -f "${btf_ids}" ]; then ${RESOLVE_BTFIDS} --patch_btfids ${btf_ids} ${ELF_FILE} @@ -121,6 +149,8 @@ cleanup() { rm -f "${ELF_FILE}.BTF.1" rm -f "${ELF_FILE}.BTF" + rm -f "${ELF_FILE}.BTF.inline" + rm -f "${ELF_FILE}.BTF.inline.link" if [ "${BTFGEN_MODE}" = "module" ]; then rm -f "${ELF_FILE}.BTF.base" rm -f "${ELF_FILE}.BTF_ids" diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index ab0b8125c8cb..5424caf5135b 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -289,7 +289,13 @@ vmlinux_link "${VMLINUX}" if is_enabled CONFIG_DEBUG_INFO_BTF; then info BTFIDS ${VMLINUX} - ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + if [ "${BTF_INLINE}" = "m" ]; then + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} \ + --btf_link .BTF.inline:btf_vmlinux_inline:vmlinux.BTF.inline \ + ${VMLINUX} + else + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + fi fi mksysmap "${VMLINUX}" System.map diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 46e80970f723..8f0933cbbe45 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -68,13 +68,13 @@ This package provides debug information for the kernel image and modules from th %undefine _debugsource_packages %undefine _debuginfo_subpackages -# Preserve .BTF and .BTF.base sections in kernel modules during debuginfo -# stripping. find-debuginfo.sh uses eu-strip which removes non-allocated ELF -# sections like .BTF by default. .BTF.base is required for BTF distillation -# support; without it, module BTF validation fails. +# Preserve .BTF, .BTF.base and .BTF.inline sections in kernel modules during +# debuginfo stripping. find-debuginfo.sh uses eu-strip which removes +# non-allocated ELF sections like .BTF by default. .BTF.base is required for +# BTF distillation support; without it, module BTF validation fails. %global with_keep_section %(%{__find_debuginfo} --help 2>&1 | grep -c keep-section) %if %{with_keep_section} -%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base +%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base --keep-section .BTF.inline %else %global _find_debuginfo_opts -r %endif From 5876080366ae3967a3aad5e193ea0e7704fa022d Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:21 +0100 Subject: [PATCH 468/660] btf: Preserve string relocation maps for split BTF Make btf_relocate() optionally return the string-offset map it builds while relocating a BTF relative to a distilled base. A subsequent split BTF object can use that map to rebase references to base strings. Also account for the accumulated base string offset when assigning a new BTF base. This is required when the base is itself split. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-5-alan.maguire@oracle.com --- include/linux/btf.h | 5 +++-- kernel/bpf/btf.c | 4 ++-- tools/lib/bpf/btf.c | 2 +- tools/lib/bpf/btf_relocate.c | 7 ++++++- tools/lib/bpf/libbpf_internal.h | 3 ++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index f731c7a5f14c..b66e20a18c54 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -608,7 +608,8 @@ struct btf_field_iter { #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids, + __u32 **map_strs); int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); __u32 *btf_field_iter_next(struct btf_field_iter *it); @@ -682,7 +683,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) } static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf, - __u32 **map_ids) + __u32 **map_ids, __u32 **map_strs) { return -EOPNOTSUPP; } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 51992cfc81c3..9569ca9b5269 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1936,7 +1936,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf_nr_types(base_btf); - btf->start_str_off = base_btf->hdr.str_len; + btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len; } static int env_resolve_init(struct btf_verifier_env *env) @@ -6997,7 +6997,7 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, goto errout; if (base_btf != vmlinux_btf) { - err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); + err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, NULL); if (err) goto errout; btf_free(base_btf); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 512b3962f75f..d2ff6f5d6d5b 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -6640,7 +6640,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) int btf__relocate(struct btf *btf, const struct btf *base_btf) { - int err = btf_relocate(btf, base_btf, NULL); + int err = btf_relocate(btf, base_btf, NULL, NULL); if (!err) btf->owns_base = false; diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index df5fa4bd87d6..e55dd75f6c95 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c @@ -441,7 +441,8 @@ static int btf_relocate_rewrite_strs(struct btf_relocate *r, __u32 i) /* If successful, output of relocation is updated BTF with base BTF pointing * at base_btf, and type ids, strings adjusted accordingly. */ -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map) +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map, + __u32 **str_map) { unsigned int nr_types = btf__type_cnt(btf); const struct btf_header *dist_base_hdr; @@ -512,6 +513,10 @@ int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map) *id_map = r.id_map; r.id_map = NULL; } + if (str_map) { + *str_map = r.str_map; + r.str_map = NULL; + } err_out: free(r.id_map); free(r.str_map); diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 546f65b95cf4..4e45dfacb548 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -249,7 +249,8 @@ const char *btf_kind_str(const struct btf_type *t); const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); const struct btf_header *btf_header(const struct btf *btf); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map, + __u32 **str_map); bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t); static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) From 42d7153fcb444b7c295560a8754a0cffafbd1c1e Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:22 +0100 Subject: [PATCH 469/660] btf: Relocate and expose module inline BTF Module .BTF.inline is split relative to its module BTF. For external modules, module BTF can instead be split relative to a distilled .BTF.base and is relocated against vmlinux when the module loads. Parse ordinary module inline BTF relative to the pre-relocation module BTF. Retain the module relocation ID and string maps, then rebase inline type IDs and string offsets onto the final vmlinux/module BTF space before exposing the result through sysfs. Malformed or unrelocatable optional inline BTF is warned about and omitted without preventing normal module BTF registration. The btf_vmlinux_inline carrier remains on its existing hash-checked, vmalloc-backed publishing path, preserving mmap support for vmlinux.inline when delivered by a module. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-6-alan.maguire@oracle.com --- kernel/bpf/btf.c | 247 ++++++++++++++++++++++++++++++++++++--------- scripts/gen-btf.sh | 7 +- 2 files changed, 203 insertions(+), 51 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9569ca9b5269..b09c04ccdc6d 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6925,51 +6925,22 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id) #ifdef CONFIG_DEBUG_INFO_BTF_MODULES -static struct btf *btf_parse_module(const char *module_name, const void *data, - unsigned int data_size, void *base_data, - unsigned int base_data_size) +static struct btf *btf_parse_split(struct btf_verifier_env *env, + const char *name, const void *data, + unsigned int data_size, struct btf *base_btf) { - struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL; - struct btf_verifier_env *env = NULL; - struct bpf_verifier_log *log; - int err = 0; - - vmlinux_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(vmlinux_btf)) - return vmlinux_btf; - if (!vmlinux_btf) - return ERR_PTR(-EINVAL); - - env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); - if (!env) - return ERR_PTR(-ENOMEM); - - log = &env->log; - log->level = BPF_LOG_KERNEL; - - if (base_data) { - base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); - if (IS_ERR(base_btf)) { - err = PTR_ERR(base_btf); - goto errout; - } - } else { - base_btf = vmlinux_btf; - } + struct btf *btf; + int err; btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); - if (!btf) { - err = -ENOMEM; - goto errout; - } + if (!btf) + return ERR_PTR(-ENOMEM); env->btf = btf; - btf->base_btf = base_btf; - btf->start_id = base_btf->nr_types; - btf->start_str_off = base_btf->hdr.str_len; + btf_set_base_btf(btf, base_btf); btf->kernel_btf = true; btf->named_start_id = 0; - strscpy(btf->name, module_name); + strscpy(btf->name, name); btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); if (!btf->data) { @@ -6996,28 +6967,180 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, if (err) goto errout; + return btf; + +errout: + btf_free(btf); + return ERR_PTR(err); +} + +static int btf_rebase_inline(struct btf *inline_btf, + const struct btf *module_btf, + const u32 *module_id_map, + const u32 *module_str_map, + u32 old_module_type_cnt) +{ + u32 old_start_id = inline_btf->start_id; + u32 old_start_str_off = inline_btf->start_str_off; + u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len; + u32 new_start_id = btf_nr_types(module_btf); + u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len; + s64 id_delta = (s64)new_start_id - old_start_id; + s64 str_delta = (s64)new_start_str_off - old_start_str_off; + u32 i; + + /* + * The inline BTF was parsed relative to the original module BTF. Its + * base IDs must therefore use the map generated when that BTF was + * relocated, while IDs for inline-local types only move by the change + * in the module BTF's starting ID. + */ + for (i = 0; i < inline_btf->nr_types; i++) { + struct btf_field_iter it; + struct btf_type *t = inline_btf->types[i]; + u32 *id, *str_off; + int err; + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + if (err) + return err; + while ((id = btf_field_iter_next(&it))) { + if (!*id) + continue; + if (*id < old_module_type_cnt) { + if (module_id_map) + *id = module_id_map[*id]; + } else if (*id >= old_start_id) { + *id += id_delta; + } else { + return -EINVAL; + } + } + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); + if (err) + return err; + while ((str_off = btf_field_iter_next(&it))) { + if (!*str_off) + continue; + if (*str_off < old_module_start_str_off) { + /* Vmlinux strings retain their offsets for in-tree modules. */ + if (!module_id_map) + continue; + if (!module_str_map || !module_str_map[*str_off]) + return -EINVAL; + *str_off = module_str_map[*str_off]; + continue; + } + *str_off += str_delta; + } + } + + btf_set_base_btf(inline_btf, module_btf); + btf_check_sorted(inline_btf); + return 0; +} + +static struct btf *btf_parse_module(const char *module_name, const void *data, + unsigned int data_size, void *base_data, + unsigned int base_data_size, + const void *inline_data, + unsigned int inline_data_size, + void **relocated_inline_data) +{ + struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL; + struct btf_verifier_env *env = NULL; + struct bpf_verifier_log *log; + u32 *module_str_map = NULL; + u32 old_module_type_cnt; + int err = 0; + + *relocated_inline_data = NULL; + vmlinux_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(vmlinux_btf)) + return vmlinux_btf; + if (!vmlinux_btf) + return ERR_PTR(-EINVAL); + + env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); + if (!env) + return ERR_PTR(-ENOMEM); + + log = &env->log; + log->level = BPF_LOG_KERNEL; + + if (base_data) { + base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); + if (IS_ERR(base_btf)) { + err = PTR_ERR(base_btf); + goto errout; + } + } else { + base_btf = vmlinux_btf; + } + + btf = btf_parse_split(env, module_name, data, data_size, base_btf); + if (IS_ERR(btf)) { + err = PTR_ERR(btf); + btf = NULL; + goto errout; + } + old_module_type_cnt = btf_nr_types(btf); + + if (inline_data_size) { + inline_btf = btf_parse_split(env, module_name, inline_data, + inline_data_size, btf); + if (IS_ERR(inline_btf)) { + pr_warn("failed to validate module [%s] inline BTF: %ld\n", + module_name, PTR_ERR(inline_btf)); + inline_btf = NULL; + } + } + if (base_btf != vmlinux_btf) { - err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, NULL); + err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, + &module_str_map); if (err) goto errout; btf_free(base_btf); base_btf = vmlinux_btf; } - btf_verifier_env_free(env); + if (inline_btf) { + err = btf_rebase_inline(inline_btf, btf, btf->base_id_map, + module_str_map, old_module_type_cnt); + if (err) { + pr_warn("failed to relocate module [%s] inline BTF: %d\n", + module_name, err); + btf_free(inline_btf); + } else { + *relocated_inline_data = inline_btf->data; + inline_btf->data = NULL; + btf_free(inline_btf); + } + } + + /* + * With a distilled base, btf_relocate() replaces the base BTF and + * rewrites string offsets. Check ordering only after that final BTF + * view has been established, so named_start_id describes the BTF used + * by name lookups. + */ btf_check_sorted(btf); + btf_verifier_env_free(env); + kvfree(module_str_map); refcount_set(&btf->refcnt, 1); return btf; errout: + kvfree(module_str_map); btf_verifier_env_free(env); + if (inline_btf) + btf_free(inline_btf); if (!IS_ERR(base_btf) && base_btf != vmlinux_btf) btf_free(base_btf); - if (btf) { - kvfree(btf->data); - kvfree(btf->types); - kfree(btf); - } + if (btf) + btf_free(btf); return ERR_PTR(err); } @@ -9119,6 +9242,8 @@ struct btf_module { struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; + struct bin_attribute *sysfs_inline_attr; + void *btf_inline_data; int flags; }; @@ -9238,6 +9363,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, struct module *mod = module; struct bin_attribute *attr; struct btf *btf; + void *inline_data = NULL, *relocated_inline_data = NULL; + unsigned int inline_data_size = 0; int err = 0; if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && @@ -9251,6 +9378,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, break; } +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + if (mod->btf_inline_data_size) { + inline_data = mod->btf_inline_data; + inline_data_size = mod->btf_inline_data_size; + } +#endif if (!mod->btf_data_size) break; btf_mod = kzalloc_obj(*btf_mod); @@ -9259,7 +9392,9 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, goto out; } btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, - mod->btf_base_data, mod->btf_base_data_size); + mod->btf_base_data, mod->btf_base_data_size, + inline_data, inline_data_size, + &relocated_inline_data); if (IS_ERR(btf)) { kfree(btf_mod); if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { @@ -9274,6 +9409,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, err = btf_alloc_id(btf); if (err) { btf_free(btf); + kvfree(relocated_inline_data); kfree(btf_mod); goto out; } @@ -9282,6 +9418,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_lock(&btf_module_mutex); btf_mod->module = module; btf_mod->btf = btf; + btf_mod->btf_inline_data = relocated_inline_data; list_add(&btf_mod->list, &btf_modules); mutex_unlock(&btf_module_mutex); @@ -9291,6 +9428,21 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, goto out; } btf_mod->sysfs_attr = attr; + + if (relocated_inline_data) { + char name[MODULE_NAME_LEN + sizeof(".inline")]; + + snprintf(name, sizeof(name), "%s.inline", mod->name); + attr = sysfs_btf_add(name, relocated_inline_data, + inline_data_size, false, NULL); + if (IS_ERR(attr)) { + kvfree(relocated_inline_data); + btf_mod->btf_inline_data = NULL; + err = 0; + break; + } + btf_mod->sysfs_inline_attr = attr; + } break; case MODULE_STATE_LIVE: if (!mod->btf_data_size) @@ -9323,6 +9475,9 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, list_del(&btf_mod->list); if (btf_mod->sysfs_attr) sysfs_btf_remove(btf_mod->sysfs_attr); + if (btf_mod->sysfs_inline_attr) + sysfs_btf_remove(btf_mod->sysfs_inline_attr); + kvfree(btf_mod->btf_inline_data); purge_cand_cache(btf_mod->btf); btf_put(btf_mod->btf); kfree(btf_mod); diff --git a/scripts/gen-btf.sh b/scripts/gen-btf.sh index e638291753d0..b17f1189298a 100755 --- a/scripts/gen-btf.sh +++ b/scripts/gen-btf.sh @@ -68,10 +68,7 @@ esac gen_btf_data() { - inline="" - if [ -n "${BTF_INLINE}" ] && [ -z "${BTF_BASE}" ]; then - inline="--inline" - fi + inline=${BTF_INLINE:+--inline} btf1="${ELF_FILE}.BTF.1" ${PAHOLE} -J ${PAHOLE_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ @@ -130,7 +127,7 @@ embed_btf_data() if [ -f "${btf_base}" ]; then ${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE} fi - btf_inline="" + btf_inline=${ELF_FILE}.BTF.inline case "${ELF_FILE}" in */btf_vmlinux_inline.ko) btf_inline=${BTF_BASE}.BTF.inline From d0de4dc51247f7f5e1a35ad5bdcc76f94e1b9095 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:23 +0100 Subject: [PATCH 470/660] selftests/bpf: Test BTF sysfs inline representations Load vmlinux, vmlinux.inline and all module and module.inline entries in /sys/fs/btf; this will allow us to sanity-check - kernel and its inline representations - in-tree modules and their inline multi-split BTF representations - out-of-tree modules and the kernel relocation done for module and module inline BTF Since bpf_testmod.ko is built as an out-of-tree module, the combination of vmlinux, normal in-tree module BTF and out-of-tree testmod ensures we sanity check relocations for each case. Also test mmap for vmlinux.inline. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-7-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/btf_sysfs.c | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c index 3923e64c4c1d..3fde8598189f 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c @@ -3,15 +3,21 @@ #include #include +#include +#include #include #include #include #include +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + static void test_btf_mmap_sysfs(const char *path, struct btf *base) { struct stat st; __u64 btf_size, end; + size_t mmap_len = 0; void *raw_data = NULL; int fd = -1; long page_size; @@ -26,6 +32,7 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) btf_size = st.st_size; end = (btf_size + page_size - 1) / page_size * page_size; + mmap_len = btf_size; fd = open(path, O_RDONLY); if (!ASSERT_GE(fd, 0, "open_btf")) @@ -39,11 +46,13 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_shared")) goto cleanup; - raw_data = mmap(NULL, end + 1, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end + 1; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_invalid_size")) goto cleanup; - raw_data = mmap(NULL, end, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_OK_PTR(raw_data, "mmap_btf")) goto cleanup; @@ -70,12 +79,84 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) cleanup: btf__free(btf); if (raw_data && raw_data != MAP_FAILED) - munmap(raw_data, btf_size); + munmap(raw_data, mmap_len); if (fd >= 0) close(fd); } +static void test_btf_inline_sysfs_all(void) +{ + struct btf *vmlinux_btf; + struct dirent *dentry; + DIR *dir; + int err = 0; + + dir = opendir(BTF_SYSFS_DIR); + if (!ASSERT_OK_PTR(dir, "open_btf_sysfs")) + return; + + vmlinux_btf = btf__parse(BTF_SYSFS_DIR "/vmlinux", NULL); + if (!ASSERT_OK_PTR(vmlinux_btf, "parse_vmlinux_btf")) { + closedir(dir); + return; + } + + while ((dentry = readdir(dir)) != NULL) { + struct btf *base_btf = NULL, *module_btf = NULL, *inline_btf = NULL; + char btf_path[PATH_MAX], inline_path[PATH_MAX]; + struct stat st; + + /* Skip ".", ".." and "foo.inline" */ + if (strstr(dentry->d_name, ".")) + continue; + + if (strcmp(dentry->d_name, "vmlinux") == 0) + base_btf = vmlinux_btf; + + if (snprintf(btf_path, sizeof(btf_path), "%s/%s", + BTF_SYSFS_DIR, dentry->d_name) >= sizeof(btf_path) || + snprintf(inline_path, sizeof(inline_path), "%s/%s%s", + BTF_SYSFS_DIR, dentry->d_name, BTF_INLINE_SUFFIX) >= + sizeof(inline_path)) { + ASSERT_FAIL("BTF sysfs path is too long\n"); + break; + } + + if (!base_btf) { + module_btf = btf__parse_split(btf_path, vmlinux_btf); + err = libbpf_get_error(module_btf); + if (err) { + /* A module can be unloaded while its sysfs entry is iterated. */ + if (err == -ENOENT) + continue; + ASSERT_OK(err, "parse_module_btf"); + continue; + } + base_btf = module_btf; + } + if (stat(inline_path, &st)) { + err = errno; + if (err != ENOENT) + ASSERT_OK(err, "stat_inline_btf"); + btf__free(module_btf); + continue; + } + if (base_btf == vmlinux_btf) + test_btf_mmap_sysfs(inline_path, base_btf); + inline_btf = btf__parse_split(inline_path, base_btf); + err = libbpf_get_error(inline_btf); + if (!err) + btf__free(inline_btf); + ASSERT_OK(err, "parse_inline_btf"); + btf__free(module_btf); + } + closedir(dir); + + btf__free(vmlinux_btf); +} + void test_btf_sysfs(void) { test_btf_mmap_sysfs("/sys/kernel/btf/vmlinux", NULL); + test_btf_inline_sysfs_all(); } From 400f0881c912af87091973f231c1b573c2b24292 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 9 Oct 2026 12:23:24 +0100 Subject: [PATCH 471/660] selftests/bpf: Add a test verifying inline information For bpf_testmod verify that we have inline info for an __always_inline'd function and it matches reasonable expectations (a single location parameter encoded in a register given that it cannot be compile-time optimized). Also verify that the offset of the LOCSEC descriptor makes sense, i.e. that it is in the range of the function where it was inlined. Because bpf_testmod is treated as an out-of-tree module, the inline information will be in btf_testmod.inline which is relocated using bpf_testmod.ko .BTF.base. Test is skipped if pahole does not encode inline info. Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009112324.1442319-8-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/btf_inline.c | 113 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 11 +- tools/testing/selftests/bpf/trace_helpers.c | 20 ++++ tools/testing/selftests/bpf/trace_helpers.h | 1 + 4 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/btf_inline.c diff --git a/tools/testing/selftests/bpf/prog_tests/btf_inline.c b/tools/testing/selftests/bpf/prog_tests/btf_inline.c new file mode 100644 index 000000000000..d88fab6429b1 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_inline.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ + +#include +#include +#include + +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + +/* + * For a specific inline site, verify we have the right function, + * loc proto and loc param representation and that the offset is + * reasonable given the caller where it was inlined. + */ +void test_btf_inline(void) +{ + struct btf *inline_btf = NULL, *btf = NULL, *vmlinux_btf = NULL; + const char *inline_caller = "bpf_testmod_test_write"; + const char *inline_func = "testmod_inline_len"; + unsigned long long caller_addr, base_addr; + bool skip = false, found_loc = false; + const struct btf_loc_param *lp; + int locsec_id, func_id, n, i; + const struct btf_type *t; + struct btf_loc *l; + const __u32 *p; + int err = 0; + + if (!env.has_testmod) { + test__skip(); + return; + } + + base_addr = module_get_base_addr("bpf_testmod"); + if (!ASSERT_NEQ(base_addr, 0, "base_addr_nonzero")) + return; + + if (!ASSERT_OK(kallsyms_find(inline_caller, &caller_addr), + "find_inline_caller")) + return; + if (!ASSERT_NEQ(caller_addr, 0, "caller_addr_nonzero")) + return; + if (!ASSERT_GT(caller_addr, base_addr, "caller_addr_gt_base_addr")) + return; + caller_addr -= base_addr; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux_btf")) + return; + + btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod", vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "bpf_testmod_btf")) + goto out; + + inline_btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod" BTF_INLINE_SUFFIX, + btf); + err = libbpf_get_error(inline_btf); + /* pahole may not have inline BTF feature support. */ + if (err == -ENOENT) { + skip = true; + goto out; + } + if (!ASSERT_OK(err, "parse_inline_btf")) + goto out; + locsec_id = btf__find_by_name_kind(inline_btf, "inline.text", BTF_KIND_LOCSEC); + if (locsec_id < 0) { + skip = true; + goto out; + } + func_id = btf__find_by_name_kind(inline_btf, inline_func, BTF_KIND_FUNC); + if (!ASSERT_GT(func_id, 0, "inline_caller_func")) + goto out; + t = btf__type_by_id(inline_btf, locsec_id); + n = btf_vlen(t); + for (i = 0, l = btf_locsec_locs(t); i < n; i++, l++) { + if (l->func == func_id) { + found_loc = true; + break; + } + } + if (!ASSERT_TRUE(found_loc, "found_loc")) + goto out; + if (!ASSERT_GT(l->loc_proto, 0, "loc_proto_id")) + goto out; + if (!ASSERT_GT(l->offset, 0, "loc_offset")) + goto out; + t = btf__type_by_id(inline_btf, l->loc_proto); + if (!ASSERT_OK_PTR(t, "loc_proto_ptr")) + goto out; + if (!ASSERT_EQ(btf_vlen(t), 1, "loc_proto_one_param")) + goto out; + p = btf_loc_proto_params(t); + if (!ASSERT_GT(*p, 0, "loc_proto_has_loc_param")) + goto out; + t = btf__type_by_id(inline_btf, *p); + if (!ASSERT_OK_PTR(t, "loc_param_ptr")) + goto out; + lp = btf_loc_param(t); + if (!ASSERT_TRUE(lp->flags & BTF_LOC_PARAM_REG, "param_has_reg")) + goto out; + if (!ASSERT_GT(l->offset, caller_addr, "inline_gt_caller")) + goto out; + /* simple sanity test to roughly ensure inline site still in function */ + if (!ASSERT_LT(l->offset, caller_addr + 512, "inline_in_caller")) + goto out; +out: + btf__free(inline_btf); + btf__free(btf); + btf__free(vmlinux_btf); + if (skip) + test__skip(); +} diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 93847ca6293b..b4f0af2aeca4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -735,6 +735,13 @@ out: EXPORT_SYMBOL(bpf_testmod_test_read); ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO); +static __always_inline size_t testmod_inline_len(size_t len) +{ + /* Keep an instruction in the inline site without changing len. */ + asm volatile("" : "+m"(len)); + return len; +} + noinline ssize_t bpf_testmod_test_write(struct file *file, struct kobject *kobj, const struct bin_attribute *bin_attr, @@ -743,7 +750,7 @@ bpf_testmod_test_write(struct file *file, struct kobject *kobj, struct bpf_testmod_test_write_ctx ctx = { .buf = buf, .off = off, - .len = len, + .len = testmod_inline_len(len), }; trace_bpf_testmod_test_write_bare_tp(current, &ctx); @@ -802,7 +809,7 @@ static struct testmod_uprobe uprobe = { .consumer.ret_handler = uprobe_ret_handler, }; -static int testmod_register_uprobe(loff_t offset) +static __always_inline int testmod_register_uprobe(loff_t offset) { int err = -EBUSY; diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 679008b310d9..f3a1d627699d 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -244,6 +244,26 @@ out: return err; } +unsigned long long module_get_base_addr(const char *module) +{ + unsigned long long addr, base_addr = 0; + char name[500]; + FILE *f; + + f = fopen("/proc/modules", "r"); + if (!f) + return 0; + + while (fscanf(f, "%499s %*s %*s %*s %*s 0x%llx", name, &addr) == 2) { + if (strcmp(module, name) == 0) { + base_addr = addr; + break; + } + } + fclose(f); + return base_addr; +} + #ifdef PROCMAP_QUERY int env_verbosity __weak = 0; diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h index 01c8ecc45627..eb0d0d9804d6 100644 --- a/tools/testing/selftests/bpf/trace_helpers.h +++ b/tools/testing/selftests/bpf/trace_helpers.h @@ -38,6 +38,7 @@ typedef int (*ksym_search_cmp_t)(const void *p1, const struct ksym *p2); int load_kallsyms(void); struct ksym *ksym_search(long key); long ksym_get_addr(const char *name); +unsigned long long module_get_base_addr(const char *module); struct ksyms *load_kallsyms_local(void); struct ksym *ksym_search_local(struct ksyms *ksyms, long key); From 87bfa22937000db29a9a09ec826ab7227eaf9e8c Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sun, 11 Oct 2026 16:48:35 +0000 Subject: [PATCH 472/660] bpf: Reject BTF ids with the same address in tracing_multi link check_dup_ids() rejects equal BTF ids, but different ids can resolve to the same address. Aliases: ffffffff8161b6b0 t __do_sys_munlockall ffffffff8161b6b0 T __ia32_sys_munlockall ffffffff8161b6b0 T __x64_sys_munlockall or several BTF_KIND_FUNC with the same name that vmlinux BTF has when CONFIG_DEBUG_INFO_BTF_INLINE is set. Each id gets its own trampoline and register_fentry_multi() adds them to data->reg with the same ip. update_ftrace_direct_add() checks that new entries are not in direct_functions, but doesn't compare them with each other, so the attach succeeds. At detach update_ftrace_direct_del() sees that 'direct' of one of the entries doesn't match and returns -EINVAL: WARNING: kernel/bpf/trampoline.c:1845 at bpf_trampoline_multi_detach+0x24c/0x280 Call Trace: bpf_link_free+0x51/0x110 bpf_link_release+0x18/0x20 __fput+0x103/0x2d0 __x64_sys_close+0x3d/0x80 The images are freed anyway, while ftrace still calls them: Oops: int3: 0000 [#1] SMP NOPTI RIP: 0010:0xffffffffc042b201 Call Trace: ? __audit_syscall_exit+0x9/0x100 ? do_syscall_64+0x5ab/0x5e0 entry_SYSCALL_64_after_hwframe+0x77/0x7f Kernel panic - not syncing: Fatal exception in interrupt Return -EINVAL from register_fentry_multi() when ip is already in data->reg. When such ids are attached via different links the second attach already fails with -EINVAL in update_ftrace_direct_add(). Move the declaration of ftrace_lookup_ip() to include/linux/ftrace.h. The lookup adds 60 msec to the attach of 64480 functions in tracing_multi_bench_attach: 1.34 -> 1.40 sec. Fixes: aef4dfa790b2 ("bpf: Add bpf_trampoline_multi_attach/detach functions")a Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20261011164836.2233874-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/ftrace.h | 1 + kernel/bpf/trampoline.c | 8 ++++++++ kernel/trace/trace.h | 3 --- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index bd76a16a63af..b041d9142d2b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -416,6 +416,7 @@ void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +struct ftrace_func_entry *ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */ diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index fc94d6062109..24edb59238f3 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1638,6 +1638,14 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + /* + * Different BTF ids get different trampolines, but they can still + * resolve to the same address (aliases, functions of the same name). + * There's only one call site to attach to. + */ + if (ftrace_lookup_ip(data->reg, ip)) + return -EINVAL; + if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 3749485a7d85..9044f03fe2d9 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1006,9 +1006,6 @@ struct ftrace_hash { struct rcu_head rcu; }; -struct ftrace_func_entry * -ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); - static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash) { return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD)); From 5f8cf2e471769374ddcfcbe2eb8c977d652de156 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sun, 11 Oct 2026 16:48:36 +0000 Subject: [PATCH 473/660] selftests/bpf: Test tracing_multi link with BTF ids of the same address The stub of a syscall without arguments is an alias of its __do_sys_ function. Find such pair where both have BTF ids. Which ones do depends on the compiler, so don't hard code the names. Check that each id can be attached and that the attach of both in one link fails with -EINVAL. With CONFIG_DEBUG_INFO_BTF_INLINE vmlinux BTF has 26 names with more than one BTF_KIND_FUNC: [117318] FUNC 'hash' type_id=76715 linkage=static [117319] FUNC 'hash' type_id=76722 linkage=static [117320] FUNC 'hash' type_id=76789 linkage=static while there is one 'hash' in kallsyms. tracing_multi_bench_attach skips the names that are not unique in kallsyms only and passes all three ids to the kernel. Take the first id of the name. Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20261011164836.2233874-3-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/tracing_multi.c | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index 0aa9532a05cf..cc727011f3a2 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -662,6 +662,97 @@ cleanup: free(ids2); } +/* + * Find two BTF ids that resolve to the same address. The stub of a syscall + * without arguments is an alias of its __do_sys_ function. + */ +static bool get_same_addr_ids(__u32 *ids) +{ + const char *prefix = "__x64_sys_"; + __u32 nr, type_id; + bool found = false; + struct btf *btf; + char alias[128]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return false; + + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr && !found; type_id++) { + const struct btf_type *type = btf__type_by_id(btf, type_id); + const char *str; + long addr; + __s32 id; + + if (!btf_is_func(type)) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str || strncmp(str, prefix, strlen(prefix))) + continue; + + snprintf(alias, sizeof(alias), "__do_sys_%s", str + strlen(prefix)); + id = btf__find_by_name_kind(btf, alias, BTF_KIND_FUNC); + if (id <= 0) + continue; + + addr = ksym_get_addr(str); + if (!addr || addr != ksym_get_addr(alias)) + continue; + + ids[0] = type_id; + ids[1] = id; + found = true; + } + + btf__free(btf); + return found; +} + +static void test_attach_same_addr(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_fail *skel; + struct bpf_link *link; + __u32 ids[2]; + int i; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + if (!get_same_addr_ids(ids)) { + test__skip(); + return; + } + + skel = tracing_multi_fail__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_fail__open_and_load")) + return; + + /* Each id attaches fine on its own.. */ + for (i = 0; i < 2; i++) { + opts.ids = &ids[i]; + opts.cnt = 1; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_single")) + goto cleanup; + bpf_link__destroy(link); + } + + /* ..but there's only one call site, so not both of them. */ + opts.ids = ids; + opts.cnt = 2; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "attach_same_addr")) + bpf_link__destroy(link); + +cleanup: + tracing_multi_fail__destroy(skel); +} + void serial_test_tracing_multi_bench_attach(void) { LIBBPF_OPTS(bpf_tracing_multi_opts, opts); @@ -748,6 +839,12 @@ void serial_test_tracing_multi_bench_attach(void) if (!btf_type_is_traceable_func(btf, type)) continue; + /* + * BTF can have more functions of the same name and they all + * resolve to the same address, take just the first one. + */ + tdelete(&str, &root, compare); + err = libbpf_ensure_mem((void **) &ids, &cap, sizeof(*ids), cnt + 1); if (err) goto cleanup; @@ -1023,6 +1120,8 @@ void test_tracing_multi_test(void) test_session(); if (test__start_subtest("attach_api_fails")) test_attach_api_fails(); + if (test__start_subtest("attach_same_addr")) + test_attach_same_addr(); RUN_TESTS(tracing_multi_verifier); if (test__start_subtest("fentry_after_multi")) test_fentry_after_multi(); From 0d5e3fc78a6f2522e6339743b8e5ecc02351a9dd Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Sat, 10 Oct 2026 15:44:35 +0800 Subject: [PATCH 474/660] bpftool: Fix out-of-bounds read in kprobe_multi link info dump show_kprobe_multi_{plain,json}() walk the sorted kallsyms array against the sorted addr/cookie array with j as a cursor into the latter. The loop terminating condition is checked after the cursor has already been used to access the array: if (j++ == info->kprobe_multi.count) break; After all count entries have been printed, j has been bumped to count, but the loop keeps running and dereferences data[count] -- one element past the end of the calloc(count, sizeof(*data)) allocation -- while comparing every remaining kallsyms symbol. If the out-of-bounds value happens to match a symbol address, a bogus entry with a garbage addr/cookie pair is printed to boot. Fix the off-by-one by advancing the cursor before the comparison, so the loop terminates as soon as the last entry has been printed. Verified with an ASAN build of bpftool against a live kprobe_multi link of 3 symbols: before: ERROR: AddressSanitizer: heap-buffer-overflow READ of size 8 at link.c:860 (show_kprobe_multi_plain) and link.c:352 (show_kprobe_multi_json), 0 bytes after the 48-byte get_addr_cookie_array() allocation after: clean output, no ASAN report Fixes: edd7f49bb884 ("bpftool: Show kprobe_multi link info") Signed-off-by: Yuan Chen Acked-by: Yafang Shao Acked-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20261010074435.1415234-1-chenyuan_fl@163.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/bpftool/link.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 088d1d206065..5cffa2d283b4 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -364,7 +364,7 @@ show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) } jsonw_uint_field(json_wtr, "cookie", data[j].cookie); jsonw_end_object(json_wtr); - if (j++ == info->kprobe_multi.count) + if (++j == info->kprobe_multi.count) break; } jsonw_end_array(json_wtr); @@ -867,7 +867,7 @@ static void show_kprobe_multi_plain(struct bpf_link_info *info) else printf(" "); - if (j++ == info->kprobe_multi.count) + if (++j == info->kprobe_multi.count) break; } error: From f6b6c87262da0ce58e9f24883cf6f7b2a3f2c338 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 12 Oct 2026 15:06:38 +0100 Subject: [PATCH 475/660] btf: Fix size/vlen validation for BTF_KIND_LOC_PARAM Validation logic allowed a BTF_KIND_LOC_PARAM with size <= 4 and vlen == 2, an inconsistent representation. Fixes: a3df11403e8c8 ("bpf: Verify BTF_KIND_LOC_PARAM vlen, flags") Suggested-by: Alexei Starovoitov Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140640.2932195-2-alan.maguire@oracle.com --- kernel/bpf/btf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index b09c04ccdc6d..64c78434fa05 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4827,12 +4827,12 @@ static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: - if (vlen > 2) + if (vlen < 1 || vlen > 2) + goto invalid_vlen; + if (!size || size > sizeof(__u64)) goto invalid_vlen; /* Ensure size/vlen are consistent */ - if (size > sizeof(__u32) && vlen != 2) - goto invalid_vlen; - if (size > sizeof(__u64)) + if (vlen != (size > sizeof(__u32) ? 2 : 1)) goto invalid_vlen; break; case BTF_LOC_PARAM_REG: From 35369ca4954c65f6423d02fad624997d72beec30 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 12 Oct 2026 15:06:39 +0100 Subject: [PATCH 476/660] selftests/bpf: Fix btf_inline to use .text section as base Relative addresses in LOCSEC are relative to the associated ELF section for the LOCSEC "inline
"; in the case of this test, it is ".text". Fix the test to compute the address via the section base rather than the overall module base address. Since addresses in BTF inline LOC_PARAMs can be module base-relative, do not remove the module base calcuation function. Fixes: 400f0881c912 ("selftests/bpf: Add a test verifying inline information") Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140640.2932195-3-alan.maguire@oracle.com --- .../selftests/bpf/prog_tests/btf_inline.c | 5 +++-- tools/testing/selftests/bpf/trace_helpers.c | 19 +++++++++++++++++++ tools/testing/selftests/bpf/trace_helpers.h | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_inline.c b/tools/testing/selftests/bpf/prog_tests/btf_inline.c index d88fab6429b1..e630e5f0f5fa 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_inline.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_inline.c @@ -32,7 +32,8 @@ void test_btf_inline(void) return; } - base_addr = module_get_base_addr("bpf_testmod"); + /* LOCSEC offsets are relative to the ELF section base. */ + base_addr = module_get_base_addr_sec("bpf_testmod", ".text"); if (!ASSERT_NEQ(base_addr, 0, "base_addr_nonzero")) return; @@ -41,7 +42,7 @@ void test_btf_inline(void) return; if (!ASSERT_NEQ(caller_addr, 0, "caller_addr_nonzero")) return; - if (!ASSERT_GT(caller_addr, base_addr, "caller_addr_gt_base_addr")) + if (!ASSERT_GE(caller_addr, base_addr, "caller_addr_ge_base_addr")) return; caller_addr -= base_addr; diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index f3a1d627699d..6d7a4b8c4ab8 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -264,6 +264,25 @@ unsigned long long module_get_base_addr(const char *module) return base_addr; } +unsigned long long module_get_base_addr_sec(const char *module, const char *sec) +{ + unsigned long long addr = 0; + char *path; + FILE *f; + + if (asprintf(&path, "/sys/module/%s/sections/%s", module, sec) < 0) + return 0; + f = fopen(path, "r"); + free(path); + if (!f) + return 0; + + if (fscanf(f, "%llx", &addr) != 1) + addr = 0; + fclose(f); + return addr; +} + #ifdef PROCMAP_QUERY int env_verbosity __weak = 0; diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h index eb0d0d9804d6..0f8d99746084 100644 --- a/tools/testing/selftests/bpf/trace_helpers.h +++ b/tools/testing/selftests/bpf/trace_helpers.h @@ -39,6 +39,7 @@ int load_kallsyms(void); struct ksym *ksym_search(long key); long ksym_get_addr(const char *name); unsigned long long module_get_base_addr(const char *module); +unsigned long long module_get_base_addr_sec(const char *module, const char *sec); struct ksyms *load_kallsyms_local(void); struct ksym *ksym_search_local(struct ksyms *ksyms, long key); From add69af5068df37c4c6e2808e9bd1332138f1180 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 12 Oct 2026 15:06:40 +0100 Subject: [PATCH 477/660] selftests/bpf: Enable CONFIG_DEBUG_INFO_BTF_INLINE for CI Ensure we add inline info to BTF for testing. Suggested-by: Alexei Starovoitov Signed-off-by: Alan Maguire Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140640.2932195-4-alan.maguire@oracle.com --- tools/testing/selftests/bpf/config | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index 2b883b388f90..d1a19fb125dd 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -23,6 +23,7 @@ CONFIG_CRYPTO_ECB=y CONFIG_CRYPTO_AES=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_INLINE=y CONFIG_DEBUG_INFO_DWARF4=y CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y From 3ead4ead078f676da5b0cda0eed1584c8d3468bd Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 5 Oct 2026 11:35:11 -0700 Subject: [PATCH 478/660] libbpf: Start v1.9 development cycle libbpf 1.8.0 has been released [1]. Update libbpf.map and libbpf_version.h to start v1.9 development cycle. [1] https://github.com/libbpf/libbpf/releases/tag/v1.8.0 Signed-off-by: Ihor Solodrai Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261005183511.201045-1-ihor.solodrai@linux.dev --- tools/lib/bpf/libbpf.map | 3 +++ tools/lib/bpf/libbpf_version.h | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 53b7b591325f..eeba4072a5da 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -473,3 +473,6 @@ LIBBPF_1.8.0 { btf__find_by_name_kind_own; btf__new_empty_opts; } LIBBPF_1.7.0; + +LIBBPF_1.9.0 { +} LIBBPF_1.8.0; diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h index c446c0cd8cf9..57b74ef3618c 100644 --- a/tools/lib/bpf/libbpf_version.h +++ b/tools/lib/bpf/libbpf_version.h @@ -4,6 +4,6 @@ #define __LIBBPF_VERSION_H #define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 8 +#define LIBBPF_MINOR_VERSION 9 #endif /* __LIBBPF_VERSION_H */ From 201a454df948021916e051b0a84a674f4f0f9671 Mon Sep 17 00:00:00 2001 From: Petr Pavlu Date: Thu, 8 Oct 2026 17:03:08 +0200 Subject: [PATCH 479/660] resolve_btfids: Fix the --fatal_warnings exit status The resolve_btfids --fatal_warnings option is intended to make the tool exit with a non-zero status if any warnings are reported. main() initializes err to -1 and uses 'goto out' to return this status when a sub-operation fails. If no error occurs, 'if (!(fatal_warnings && warnings)) err = 0;' modifies the final status depending on whether any warnings have been reported. Commit 522397d05e7d ("resolve_btfids: Change in-place update with raw binary output") broke this logic by assigning the return values of newly added sub-operations to err. As a result, successful sub-operations overwrite the initial err value of -1 with 0, causing the tool to exit successfully even when --fatal_warnings is enabled and warnings have been reported. Fix the issue by setting the final value of err unconditionally, so the code doesn't depend on its previous value. Fixes: 522397d05e7d ("resolve_btfids: Change in-place update with raw binary output") Signed-off-by: Petr Pavlu Signed-off-by: Daniel Borkmann Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20261008-resolve_btfids-warnings-v1-1-0d4e613ffa6e@suse.com --- tools/bpf/resolve_btfids/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index f737fac60fa2..5ad547356303 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -2210,8 +2210,7 @@ dump_btf: goto out; } - if (!(fatal_warnings && warnings)) - err = 0; + err = fatal_warnings && warnings ? -1 : 0; out: free_btf_links(&btf_links); /* From e980770b431a7d459da4eca37c0300ca07058e54 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:36 -0700 Subject: [PATCH 480/660] bpf: More precise stack liveness effects for global func params The following types can be returned by the btf_prepare_func_args(): - ARG_PTR_TO_CTX - ARG_PTR_TO_BTF_ID (+ flags) - ARG_PTR_TO_MEM (+ flags) - ARG_PTR_TO_ARENA - ARG_PTR_TO_DYNPTR - ARG_SCALAR Of these, valid programs can pass frame pointer only as ARG_PTR_TO_{MEM,BTF_ID,DYNPTR}. Additionally, pointer arithmetic allows the following trick: 0: (bf) r1 = r10 1: (07) r1 += -8 2: (1f) r1 -= r10 3: (85) call pc+1 // r1=fp0 ? 4: (95) exit This commit changes bpf_global_subprog_stack_access_bytes() to handle ARG_SCALAR and ARG_PTR_TO_DYNPTR, thus addressing the full set of possible returned values. Fixes: 8212c1c2986d ("bpf: Derive global subprog stack liveness from BTF signatures") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-1-6d9d61965486@gmail.com --- kernel/bpf/verifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2f190abe3448..e477ef4465d5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14871,7 +14871,7 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_i switch (base_type(sub->args[arg].arg_type)) { case ARG_UNUSED: /* Not an argument slot. */ - case ARG_ANYTHING: /* A scalar, the callee can't dereference it. */ + case ARG_SCALAR: /* A scalar, the callee can't dereference it. */ return (struct arg_access_info) {}; case ARG_PTR_TO_BTF_ID: /* @@ -14887,6 +14887,10 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_i info.must_write = true; } return info; + case ARG_PTR_TO_DYNPTR: + /* The callee may adjust the view, but cannot reinitialize the dynptr. */ + info.size = BPF_DYNPTR_SIZE; + return info; default: return info; } From 06e0e41e6223dbe886ee986550c0ffa06003f482 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:37 -0700 Subject: [PATCH 481/660] bpf: Avoid duplicate declaration for BPF_SCEV_REGS_NUM The "bpf: Use SCEV to widen bounded loops" commit introduced a second (identical and unnecessary) definition for the BPF_SCEV_REGS_NUM macro. Fixes: 088a5477a9bd ("bpf: Avoid widening registers that hinder exact stack-slot tracking") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-2-6d9d61965486@gmail.com --- include/linux/bpf_verifier.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 54cb6d3d0653..791398e14962 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -708,9 +708,6 @@ struct bpf_loop_exit { int to; /* instruction outside the loop */ }; -/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ -#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) - struct bpf_loop { struct bpf_backedge backedges[MAX_BACKEDGES]; /* edges exiting from this loop, includes edges from nested loops */ From 4d274f1ef04949a2ec46a62c0a5e2ecde668169a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:38 -0700 Subject: [PATCH 482/660] bpf: Reg_bounds_sync() comment fixup Array traversal logic in btf_struct_access() starts from reg_smin(). Update comment to match the logic. Fixes: 0892ef2a7ef3 ("bpf: Representation for intervals with steps") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-3-6d9d61965486@gmail.com --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e477ef4465d5..ec136cf377c1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2354,7 +2354,7 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) * potentially w/o intersection, or intersecting after N * tnum_step()'s, where N can be large. On the other hand, * step+base *must* be in sync with r64 in order for - * array traversal logic starting at reg_umin() to be sound. + * array traversal logic starting at reg_smin() to be sound. * Hence, place this as a final step in the reg_bounds_sync(). */ __update_bounds_from_step(reg); From ff9e7261e0ce7272c6bf6e147ac39b56e8940987 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:39 -0700 Subject: [PATCH 483/660] selftests/bpf: Stack effects of scalar/dynptr global subprog params Verify that a frame pointer derived value does not induce stack liveness effects, when passed as a scalar argument of a global function. Verify that a stack-based dynptr passed to a global function marks live only corresponding 16 bytes. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-4-6d9d61965486@gmail.com --- .../selftests/bpf/progs/verifier_live_stack.c | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 3d88cb28b8fe..d2fdc65c84c5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -182,6 +182,62 @@ __naked void global_callee_arg(void) ::: __clobber_all); } +__noinline int global_int_arg(int x) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +__msg("stack use/def subprog#0 global_noop_int_arg (d0,cs0):") +__msg_next(" 0: (bf) r1 = r10{{$}}") +__msg_next(" 1: (07) r1 += -8{{$}}") +__msg_next(" 2: (1f) r1 -= r10{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit") +__naked void global_noop_int_arg(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r1 -= r10;" /* This is a scalar now. */ + "call global_int_arg;" /* And should not be tracked as stack effect. */ + "exit;" + ::: __clobber_all); +} + +static char global_dynptr_buf[8]; + +__noinline int global_noop_dynptr_arg(struct bpf_dynptr *p) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* A global dynptr argument may read/write only its 16 bytes, with no definite write. */ +__msg("stack use/def subprog#0 global_callee_dynptr_arg (d0,cs0):") +__msg(" 7: (bf) r1 = r10{{$}}") +__msg_next(" 8: (07) r1 += -32{{$}}") +__msg_next(" 9: (85) call pc+1{{ +}}; use: fp0-24 fp0-32 may_def: fp0-24 fp0-32{{$}}") +__msg_next(" 10: (95) exit{{$}}") +__naked void global_callee_dynptr_arg(void) +{ + asm volatile ( + "r1 = %[global_dynptr_buf] ll;" + "r2 = 8;" + "r3 = 0;" + "r4 = r10;" + "r4 += -32;" + "call %[bpf_dynptr_from_mem];" + "r1 = r10;" + "r1 += -32;" + "call global_noop_dynptr_arg;" + "exit;" + :: __imm_addr(global_dynptr_buf), __imm(bpf_dynptr_from_mem) + : __clobber_all); +} + struct task_struct { int pid; }; __noinline int global_noop_btf_arg(struct task_struct *p __arg_untrusted) From 3f3137ef1c9e1c8b800e0ec991156e463dbeb8bf Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:40 -0700 Subject: [PATCH 484/660] selftests/bpf: Harden __msg pattern for global_callee_btf_arg The original pattern: __msg("3: (85) call pc+{{[0-9]+}}\n") Is ambigous, as this line is printed on the main verification pass as well. Harden the test case by ensuring a __msg_next sequence matching the stack use/def results specifically. Fixes: 063c85f5d8c7 ("selftests/bpf: Tests for BTF-based global subprog stack liveness") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-5-6d9d61965486@gmail.com --- tools/testing/selftests/bpf/progs/verifier_live_stack.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index d2fdc65c84c5..5f9b11a81260 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -249,7 +249,11 @@ SEC("socket") __success __log_level(2) /* Untrusted BTF pointer arguments need no caller stack state. */ __msg("stack use/def subprog#0 global_callee_btf_arg (d0,cs0):") -__msg("3: (85) call pc+{{[0-9]+}}\n") +__msg_next(" 0: (7a) *(u64 *)(r10 -8) = 0{{.*}}") +__msg_next(" 1: (bf) r1 = r10{{$}}") +__msg_next(" 2: (07) r1 += -8{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit{{$}}") __naked void global_callee_btf_arg(void) { asm volatile ( From 1bb030a11fa6179f317880d3ec348361af9a63c3 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:41 -0700 Subject: [PATCH 485/660] selftests/bpf: Harden __msg patterns for step_movsx_bounds_on_line The `{{.*}}` in the pattern for R5 below might hide the `step=...` logging from the __no_step annotation. Replace it with full expected log instead. Also drop __no_step for R3 above, where it is unnecessary. Fixes: 2db5837f9ada ("selftests/bpf: Tests for register base/step arithmetic") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-6-6d9d61965486@gmail.com --- tools/testing/selftests/bpf/progs/verifier_bounds_step.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c index c9ee1c7efcbb..d824a5d48a39 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c @@ -338,9 +338,9 @@ SEC("socket") __success __log_level(2) __msg("r1 = (s16)r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") __msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)") -__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") __no_step +__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") __msg("w4 = (s16)w1 {{.*}}R4=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") -__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,{{.*}})") __no_step +__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,var_off=(0x0; 0xffffffff))") __naked void step_movsx_bounds_on_line(void) { asm volatile (" \ From 75fc921e7f80864587f915d9050b84afe564cef0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Oct 2026 23:24:42 -0700 Subject: [PATCH 486/660] selftests/bpf: BPF_F_ANY_ALIGNMENT for byte_array_size_spanning() The test case indexes inside `char bytes[32]` array with unaligned offset. Use BPF_F_ANY_ALIGNMENT flag to avoid the test rejection for unrelated reason. Fixes: e187fac8ac6b ("selftests/bpf: Tests for varying offset access to PTR_TO_BTF_ID") Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013-scev-minimal-v3-followups-v2-7-6d9d61965486@gmail.com --- tools/testing/selftests/bpf/progs/verifier_btf_array_access.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c index be001cc77f81..9dc2c9a5f8b7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c @@ -114,6 +114,7 @@ int long_array_in_bounds(void *ctx) */ SEC("syscall") __failure +__flag(BPF_F_ANY_ALIGNMENT) __msg("invalid variable offset access into struct outer") int byte_array_size_spanning(void *ctx) { From 63d4c99d25aeb163e96ae9a5a6dfbfac316beaf4 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:13 +0200 Subject: [PATCH 487/660] bpf, x86: Drop redundant REX prefixes from zero and sign extensions movzx into a 32-bit register already clears the upper 32 bits, so the REX.W prefix on the 8 and 16-bit BPF_LDX loads only costs a byte. This also matches what the arena variants of these loads emit. Likewise, the 32-bit movsx from an 8-bit register only needs a REX prefix when the destination is an extended register or the source is sil, dil, bpl or an extended register. LLVM emits these loads for u8 and u16 fields and, with cpu v4, the movsx for a cast to s8 or s16. Before/after for r0 = *(u8 *)(r1 + 0): 48 0f b6 47 00 movzbq 0x0(%rdi),%rax 0f b6 47 00 movzbl 0x0(%rdi),%eax Before/after for w1 = (s8)w0: 40 0f be f8 movsbl %al,%edi 0f be f8 movsbl %al,%edi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 083fcd6cf15b..adec4f8e422e 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1106,8 +1106,9 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, } else { /* movs[b,w]l dst, src */ if (num_bits == 8) { - EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe, - add_2reg(0xC0, src_reg, dst_reg)); + if (is_ereg(dst_reg) || is_ereg_8l(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0f, 0xbe, add_2reg(0xC0, src_reg, dst_reg)); } else if (num_bits == 16) { if (is_ereg(dst_reg) || is_ereg(src_reg)) EMIT1(add_2mod(0x40, src_reg, dst_reg)); @@ -1273,12 +1274,16 @@ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) switch (size) { case BPF_B: - /* Emit 'movzx rax, byte ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); + /* Emit 'movzx eax, byte ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB6); break; case BPF_H: - /* Emit 'movzx rax, word ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); + /* Emit 'movzx eax, word ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB7); break; case BPF_W: /* Emit 'mov eax, dword ptr [rax+0x14]' */ From 44df4d88aa6351fcb563b6e7aa4f9964d7354bca Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:14 +0200 Subject: [PATCH 488/660] bpf, x86: Omit the displacement byte for zero offsets A memory operand with a zero offset was encoded with a mod 01 ModR/M and an explicit zero displacement byte. Use the mod 00 form instead, which saves a byte for every load, store and atomic with a zero offset, as LLVM emits them for the first field of a struct and for every plain pointer dereference. The exception are rbp and r13 as base register, where mod 00 with r/m 101 selects a RIP-relative disp32 and with SIB base 101 a disp32 without base, so these keep their displacement byte. Also let emit_st() reuse emit_insn_suffix() instead of open-coding the ModR/M encoding. Before/after for r0 = *(u64 *)(r1 + 0): 48 8b 47 00 movq 0x0(%rdi),%rax 48 8b 07 movq (%rdi),%rax Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-3-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index adec4f8e422e..70243ccbe423 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1120,18 +1120,26 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, *pprog = prog; } +/* + * rbp and r13 as base register always need a displacement, since mod 00 + * with r/m 101 encodes a RIP-relative disp32 and with SIB base 101 a + * disp32 without base instead. + */ +static bool base_needs_disp(u32 reg) +{ + return reg2hex[reg] == 5; +} + /* Emit the suffix (ModR/M etc) for addressing *(ptr_reg + off) and val_reg */ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) { u8 *prog = *pprog; - if (is_imm8(off)) { - /* 1-byte signed displacement. - * - * If off == 0 we could skip this and save one extra byte, but - * special case of x86 R13 which always needs an offset is not - * worth the hassle - */ + if (!off && !base_needs_disp(ptr_reg)) { + /* No displacement */ + EMIT1(add_2reg(0x00, ptr_reg, val_reg)); + } else if (is_imm8(off)) { + /* 1-byte signed displacement */ EMIT2(add_2reg(0x40, ptr_reg, val_reg), off); } else { /* 4-byte signed displacement */ @@ -1144,7 +1152,10 @@ static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index { u8 *prog = *pprog; - if (is_imm8(off)) { + if (!off && !base_needs_disp(ptr_reg)) { + EMIT2(add_2reg(0x04, BPF_REG_0, val_reg), + add_2reg(0, ptr_reg, index_reg) /* SIB */); + } else if (is_imm8(off)) { EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); } else { EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); @@ -1507,11 +1518,7 @@ static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, break; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); - + emit_insn_suffix(&prog, dst_reg, BPF_REG_0, insn_off); EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); *pprog = prog; } From c4da703867edd93a9808ac4c38659ac34d70baa0 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:15 +0200 Subject: [PATCH 489/660] bpf, x86: Shorten the stack adjusts and the tail call counter loads The prologue and the tail call sequences always used the imm32 forms of sub/add rsp and a disp32 load of the tail call counter pointer, even when the rounded stack depth fits into a byte. Route them through the existing emit_sub_rsp()/emit_add_rsp() and emit_ldx() helpers which pick the short forms, and derive the ip adjustment for the counter load in front of a subprog call from the emitted length instead of hardcoding the 7 bytes. Nothing depends on the size of these instructions: the tail call entry offset covers only the code in front of the stack adjustment, and the poke targets are computed from the emitted bytes. Before/after for a program with 32 bytes of stack: 48 81 ec 20 00 00 00 subq $0x20,%rsp 48 83 ec 20 subq $0x20,%rsp 48 8b 85 d0 ff ff ff movq -0x30(%rbp),%rax 48 8b 45 d0 movq -0x30(%rbp),%rax Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 70243ccbe423..c6e814f7c806 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -421,7 +421,7 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) } /* add rsp, depth */ -static void emit_add_rsp(u8 **pprog, u16 depth) +static void emit_add_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -435,7 +435,7 @@ static void emit_add_rsp(u8 **pprog, u16 depth) } /* sub rsp, depth */ -static void emit_sub_rsp(u8 **pprog, u16 depth) +static void emit_sub_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -472,6 +472,7 @@ static void emit_nops(u8 **pprog, int len) * in arch/x86/kernel/alternative.c */ static int emit_call(u8 **prog, void *func, void *ip); +static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off); static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity) { @@ -607,8 +608,7 @@ static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_from_cb EMIT_ENDBR(); /* sub rsp, rounded_stack_depth */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); + emit_sub_rsp(&prog, round_up(stack_depth, 8)); if (tail_call_reachable) emit_prologue_tail_call(&prog, is_subprog); *pprog = prog; @@ -851,7 +851,8 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_indirect_label - (prog + 2 - start); @@ -888,9 +889,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ - round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); /* add rsp, sd */ /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ @@ -923,7 +922,8 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_direct_label - (prog + 2 - start); @@ -955,8 +955,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); emit_nops(&prog, X86_PATCH_SIZE); @@ -1857,7 +1856,7 @@ static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) #define __LOAD_TCC_PTR(off) \ - EMIT3_off32(0x48, 0x8B, 0x85, off) + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, off) /* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */ #define LOAD_TAIL_CALL_CNT_PTR(stack) \ __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack)) @@ -2958,8 +2957,10 @@ populate_extable: func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { + u8 *tcc_load = prog; + LOAD_TAIL_CALL_CNT_PTR(stack_depth); - ip += 7; + ip += prog - tcc_load; } if (!imm32) return -EINVAL; From 649caad437d77e86f92f8e8394a22c4370cf7254 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:16 +0200 Subject: [PATCH 490/660] bpf, x86: Use shorter test and cmp encodings for immediates test has no sign-extended imm8 form, so a JSET against a constant was always emitted as a 6 to 7 byte test with imm32. When the mask fits into the low byte, the result only depends on that byte, so test the 8-bit register instead, which takes 2 to 4 bytes. Reading the low byte of a register that was written as a whole has no partial register penalty on Intel or AMD cores. Also use the one byte shorter accumulator forms of test and cmp with imm32 when the destination is r0, like the ALU immediates already do. Before/after for if r1 & 0x40 goto ...: 48 f7 c7 40 00 00 00 testq $0x40,%rdi 40 f6 c7 40 testb $0x40,%dil Before/after for if r0 == 0x1000 goto ...: 48 81 f8 00 10 00 00 cmpq $0x1000,%rax 48 3d 00 10 00 00 cmpq $0x1000,%rax Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-5-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c6e814f7c806..079703c173a5 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3080,10 +3080,26 @@ populate_extable: case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: + if ((u32)imm32 <= U8_MAX) { + /* + * test dst_reg8, imm8: with the upper bits of the + * mask clear the result only depends on the low byte. + */ + if (is_ereg_8l(dst_reg)) + EMIT1(add_1mod(0x40, dst_reg)); + if (is_axreg(dst_reg)) + EMIT2(0xA8, imm32); + else + EMIT3(0xF6, add_1reg(0xC0, dst_reg), imm32); + goto emit_cond_jmp; + } /* test dst_reg, imm32 */ maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); + if (is_axreg(dst_reg)) + EMIT1_off32(0xA9, imm32); + else + EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); goto emit_cond_jmp; case BPF_JMP | BPF_JEQ | BPF_K: @@ -3120,6 +3136,8 @@ populate_extable: if (is_imm8(imm32)) EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); + else if (is_axreg(dst_reg)) + EMIT1_off32(0x3D, imm32); else EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); From fa6979fde9f679cef63fef5aff13c382227d4570 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:17 +0200 Subject: [PATCH 491/660] bpf, x86: Compare against 0, 127 and -128 with the adjusted condition A strict compare against 1 or 128 is the non-strict compare against 0 or 127, and a strict compare against -129 or, with a signed condition, -1 the non-strict compare against -128 or 0. LLVM canonicalizes x <= 0 to x < 1, x >= 0 to x > -1 and x <= 127 to x < 128, so these reach the JIT as compares against 1, -1 and 128. Adjust the condition and the immediate, so that the compare against 0 becomes the test without an immediate already emitted for it and 127 and -128 fit into an imm8. Before, for if r1 s> -1 goto ... and if r1 < 128 goto ...: 48 83 ff ff cmpq $-0x1,%rdi 7f xx jg ... 48 81 ff 80 00 00 00 cmpq $0x80,%rdi 72 xx jb ... After: 48 85 ff testq %rdi,%rdi 7d xx jge ... 48 83 ff 7f cmpq $0x7f,%rdi 76 xx jbe ... Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-6-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 65 +++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 079703c173a5..70473cd43aa8 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2011,6 +2011,54 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, return prog - start; } +/* + * A strict compare against 1 or 128 is the non-strict one against 0 or + * 127, a strict one against -129 or, with a signed condition, -1 the + * non-strict one against -128 or 0: the compare against 0 is a test + * without immediate, 127 and -128 fit into an imm8. + */ +static void adjust_cmp_imm(u32 *op, s32 *imm) +{ + if (*imm == 1 || *imm == 128) { + switch (*op) { + case BPF_JLT: + *op = BPF_JLE; + break; + case BPF_JGE: + *op = BPF_JGT; + break; + case BPF_JSLT: + *op = BPF_JSLE; + break; + case BPF_JSGE: + *op = BPF_JSGT; + break; + default: + return; + } + *imm -= 1; + } else if (*imm == -1 || *imm == -129) { + switch (*op) { + case BPF_JGT: + case BPF_JLE: + /* -1 is the largest unsigned value */ + if (*imm == -1) + return; + *op = *op == BPF_JGT ? BPF_JGE : BPF_JLT; + break; + case BPF_JSGT: + *op = BPF_JSGE; + break; + case BPF_JSLE: + *op = BPF_JSLT; + break; + default: + return; + } + *imm += 1; + } +} + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -2153,6 +2201,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u8 jmp_cond; u8 *func; int nops; + u32 jmp_op = BPF_OP(insn->code); + s32 cmp_imm; if (priv_frame_ptr) { if (src_reg == BPF_REG_FP) @@ -3122,8 +3172,11 @@ populate_extable: case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: + cmp_imm = imm32; + adjust_cmp_imm(&jmp_op, &cmp_imm); + /* test dst_reg, dst_reg to save one extra byte */ - if (imm32 == 0) { + if (cmp_imm == 0) { maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); @@ -3134,15 +3187,15 @@ populate_extable: maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); + if (is_imm8(cmp_imm)) + EMIT3(0x83, add_1reg(0xF8, dst_reg), cmp_imm); else if (is_axreg(dst_reg)) - EMIT1_off32(0x3D, imm32); + EMIT1_off32(0x3D, cmp_imm); else - EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); + EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), cmp_imm); emit_cond_jmp: /* Convert BPF opcode to x86 */ - switch (BPF_OP(insn->code)) { + switch (jmp_op) { case BPF_JEQ: jmp_cond = X86_JE; break; From 135b45cf05b9fc0d7cc22a433608fd40d87894d3 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:18 +0200 Subject: [PATCH 492/660] bpf, x86: Use movzx for byte and word masks and a not for xor with -1 LLVM lowers u8 and u16 truncations to r &= 0xff and r &= 0xffff, and a bitwise not to r ^= -1, since BPF has neither a zero extending move from a sub-register nor a not instruction. On x86 these masks need the imm32 form of and as they do not fit into a sign-extended imm8. Emit a movzx from the low byte or word instead, which is 3 to 4 bytes and zero extends into the full register for both the 32 and 64-bit variant, and a not for the xor with -1. Both are single cycle instructions on Intel and AMD cores. Before/after for r1 &= 0xff, r1 &= 0xffff and r1 ^= -1: 48 81 e7 ff 00 00 00 andq $0xff,%rdi 40 0f b6 ff movzbl %dil,%edi 48 81 e7 ff ff 00 00 andq $0xffff,%rdi 0f b7 ff movzwl %di,%edi 48 83 f7 ff xorq $-1,%rdi 48 f7 d7 notq %rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-7-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 70473cd43aa8..e7439fb5037d 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1129,6 +1129,20 @@ static bool base_needs_disp(u32 reg) return reg2hex[reg] == 5; } +static void emit_movzx_reg(u8 **pprog, int num_bits, u32 dst_reg, u32 src_reg) +{ + u8 *prog = *pprog; + + /* movz[b,w]l dst, src */ + if (is_ereg(dst_reg) || + (num_bits == 8 ? is_ereg_8l(src_reg) : is_ereg(src_reg))) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0F, num_bits == 8 ? 0xB6 : 0xB7, + add_2reg(0xC0, src_reg, dst_reg)); + + *pprog = prog; +} + /* Emit the suffix (ModR/M etc) for addressing *(ptr_reg + off) and val_reg */ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) { @@ -2312,6 +2326,21 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: + if (BPF_OP(insn->code) == BPF_AND && + (imm32 == 0xff || imm32 == 0xffff)) { + /* movzx dst_reg, dst_reg8/16 */ + emit_movzx_reg(&prog, imm32 == 0xff ? 8 : 16, + dst_reg, dst_reg); + break; + } + if (BPF_OP(insn->code) == BPF_XOR && imm32 == -1) { + /* not dst_reg */ + maybe_emit_1mod(&prog, dst_reg, + BPF_CLASS(insn->code) == BPF_ALU64); + EMIT2(0xF7, add_1reg(0xD0, dst_reg)); + break; + } + maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); @@ -2573,11 +2602,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Emit 'movzwl eax, ax' to zero extend 16-bit * into 64 bit */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + emit_movzx_reg(&prog, 16, dst_reg, dst_reg); break; case 32: /* Emit 'mov eax, eax' to clear upper 32-bits */ From 707e1331f52b0fc58aa517ea1da4e40c2243b2aa Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:19 +0200 Subject: [PATCH 493/660] bpf, x86: Consolidate the ALU, shift and neg encodings into helpers The add, sub, and, or and xor with an immediate or a register, the shifts by an immediate and neg are open coded in the ALU cases of do_jit(), the add with an immediate once more in the tail call and KASAN check sequences. Add emit_alu_imm(), emit_alu_reg(), emit_shift_imm() and emit_neg() for them and use them there, together with emit_mov_reg(), emit_mov_imm32() and emit_add_rsp() for the open coded moves and stack adjustment in the tail call, the division and the trampoline. The emitted code is unchanged. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-8-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 144 +++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 66 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e7439fb5037d..3e367600a0ad 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -473,6 +473,8 @@ static void emit_nops(u8 **pprog, int len) */ static int emit_call(u8 **prog, void *func, void *ip); static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off); +static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg); +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64); static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity) { @@ -840,7 +842,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if (index >= array->map.max_entries) * goto out; */ - EMIT2(0x89, 0xD2); /* mov edx, edx */ + emit_mov_reg(&prog, false, BPF_REG_3, BPF_REG_3); /* mov edx, edx */ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ offsetof(struct bpf_array, map.max_entries)); @@ -894,8 +896,8 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ offsetof(struct bpf_prog, bpf_func)); - EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ - X86_TAIL_CALL_OFFSET); + /* add rcx, X86_TAIL_CALL_OFFSET */ + emit_alu_imm(&prog, BPF_ADD, BPF_REG_4, X86_TAIL_CALL_OFFSET, true); /* * Now we're ready to jump into next BPF program * rdi == ctx (1st arg) @@ -1204,6 +1206,67 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +/* add/sub/and/or/xor dst_reg, imm with the imm8 and the rax short forms */ +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64) +{ + /* ModR/M opcode extension and rax short form per operation */ + static const u8 modrm[] = { + [BPF_ADD] = 0xC0, [BPF_SUB] = 0xE8, [BPF_AND] = 0xE0, + [BPF_OR] = 0xC8, [BPF_XOR] = 0xF0, + }; + static const u8 short_op[] = { + [BPF_ADD] = 0x05, [BPF_SUB] = 0x2D, [BPF_AND] = 0x25, + [BPF_OR] = 0x0D, [BPF_XOR] = 0x35, + }; + u8 *prog = *pprog; + + maybe_emit_1mod(&prog, dst_reg, is64); + if (is_imm8(imm)) + EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm); + else if (is_axreg(dst_reg)) + EMIT1_off32(short_op[op], imm); + else + EMIT2_off32(0x81, add_1reg(modrm[op], dst_reg), imm); + + *pprog = prog; +} + +/* add/sub/and/or/xor dst_reg, src_reg */ +static void emit_alu_reg(u8 **pprog, u32 op, u32 dst_reg, u32 src_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_mod(&prog, dst_reg, src_reg, is64); + EMIT2(simple_alu_opcodes[op], add_2reg(0xC0, dst_reg, src_reg)); + + *pprog = prog; +} + +/* shl/shr/sar reg, imm */ +static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_1mod(&prog, reg, is64); + if (imm == 1) + EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); + else + EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + + *pprog = prog; +} + +/* neg dst_reg */ +static void emit_neg(u8 **pprog, u32 dst_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_1mod(&prog, dst_reg, is64); + EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + + *pprog = prog; +} + static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, u32 addr_reg, s32 off, u32 bpf_size, u8 *ip, bool is_write) @@ -1258,15 +1321,8 @@ static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, EMIT_mov(BPF_REG_1, addr_reg); /* add rdi, off (if offset is non-zero) */ - if (off) { - if (is_imm8(off)) { - /* add rdi, imm8 */ - EMIT4(0x48, 0x83, 0xC7, (u8)off); - } else { - /* add rdi, imm32 */ - EMIT3_off32(0x48, 0x81, 0xC7, off); - } - } + if (off) + emit_alu_imm(&prog, BPF_ADD, BPF_REG_1, off, true); /* Adjust ip to account for the instrumentation generated so far */ ip += (prog - *pprog); @@ -2207,7 +2263,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; bool accesses_stack_only; - u8 b2 = 0, b3 = 0; + u8 b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; @@ -2247,10 +2303,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: - maybe_emit_mod(&prog, dst_reg, src_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - b2 = simple_alu_opcodes[BPF_OP(insn->code)]; - EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); + emit_alu_reg(&prog, BPF_OP(insn->code), dst_reg, src_reg, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_X: @@ -2311,9 +2365,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * /* neg dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + emit_neg(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_ADD | BPF_K: @@ -2341,42 +2393,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; } - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - /* - * b3 holds 'normal' opcode, b2 short form only valid - * in case dst is eax/rax. - */ - switch (BPF_OP(insn->code)) { - case BPF_ADD: - b3 = 0xC0; - b2 = 0x05; - break; - case BPF_SUB: - b3 = 0xE8; - b2 = 0x2D; - break; - case BPF_AND: - b3 = 0xE0; - b2 = 0x25; - break; - case BPF_OR: - b3 = 0xC8; - b2 = 0x0D; - break; - case BPF_XOR: - b3 = 0xF0; - b2 = 0x35; - break; - } - - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(b3, dst_reg), imm32); - else if (is_axreg(dst_reg)) - EMIT1_off32(b2, imm32); - else - EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); + emit_alu_imm(&prog, BPF_OP(insn->code), dst_reg, imm32, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -2429,7 +2447,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * xor edx, edx * equivalent to 'xor rdx, rdx', but one byte less */ - EMIT2(0x31, 0xd2); + emit_mov_imm32(&prog, false, BPF_REG_3, 0); /* div src_reg */ maybe_emit_1mod(&prog, src_reg, is64); @@ -2493,14 +2511,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - b3 = simple_alu_opcodes[BPF_OP(insn->code)]; - if (imm32 == 1) - EMIT2(0xD1, add_1reg(b3, dst_reg)); - else - EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); + emit_shift_imm(&prog, BPF_OP(insn->code), dst_reg, + imm32, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_LSH | BPF_X: @@ -4206,7 +4218,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ - EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ + emit_add_rsp(&prog, 8); /* add rsp, 8 */ } emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ From d59f47e031aecdb6d8e03d4d4d8880613339f086 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:20 +0200 Subject: [PATCH 494/660] bpf, x86: Use shorter and faster forms for some ALU immediates An add or sub of 1 or -1 is an inc or dec, which is a byte shorter and a single uop on all current cores, and the flags they leave untouched are not read across BPF instructions anyway. An add of 128 is the sub of -128 and the other way round, which fits into an imm8. An and of a 64-bit register with a non-negative immediate clears the upper half just like the 32-bit and does, which drops the REX.W prefix on the lower eight registers. A shift left by one is the add of the register to itself, the same size and a single uop that runs on more ports. LLVM emits all of them, r += 1 for every counter, r &= mask for the masks beyond the byte and word ones and r <<= 1 for x * 2 and x + x, and does the same replacements for native code. Before, for r1 += 1, r1 += 128, r1 &= 0x3ff and r1 <<= 1: 48 83 c7 01 addq $0x1,%rdi 48 81 c7 80 00 00 00 addq $0x80,%rdi 48 81 e7 ff 03 00 00 andq $0x3ff,%rdi 48 d1 e7 shlq %rdi After: 48 ff c7 incq %rdi 48 83 ef 80 subq $-0x80,%rdi 81 e7 ff 03 00 00 andl $0x3ff,%edi 48 01 ff addq %rdi,%rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-9-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 3e367600a0ad..1c84d2a02aba 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1219,9 +1219,22 @@ static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64) [BPF_OR] = 0x0D, [BPF_XOR] = 0x35, }; u8 *prog = *pprog; + bool inc; + /* A non-negative mask clears the upper half like the 32-bit and does */ + if (op == BPF_AND && imm >= 0) + is64 = false; + /* add 128 is sub -128 and the other way round, which fit into an imm8 */ + if ((op == BPF_ADD || op == BPF_SUB) && imm == 128) { + op = op == BPF_ADD ? BPF_SUB : BPF_ADD; + imm = -128; + } + inc = op == BPF_ADD ? imm == 1 : imm == -1; maybe_emit_1mod(&prog, dst_reg, is64); - if (is_imm8(imm)) + /* inc or dec for +-1, the flags are not read across instructions */ + if ((op == BPF_ADD || op == BPF_SUB) && (imm == 1 || imm == -1)) + EMIT2(0xFF, add_1reg(inc ? 0xC0 : 0xC8, dst_reg)); + else if (is_imm8(imm)) EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm); else if (is_axreg(dst_reg)) EMIT1_off32(short_op[op], imm); @@ -1247,11 +1260,16 @@ static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64) { u8 *prog = *pprog; - maybe_emit_1mod(&prog, reg, is64); - if (imm == 1) - EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); - else - EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + /* The shift left by one is an add of the register to itself */ + if (op == BPF_LSH && imm == 1) { + emit_alu_reg(&prog, BPF_ADD, reg, reg, is64); + } else { + maybe_emit_1mod(&prog, reg, is64); + if (imm == 1) + EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); + else + EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + } *pprog = prog; } From 0847f6cb2e6aec22e5a7bf4e3000e9a080bf9878 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:21 +0200 Subject: [PATCH 495/660] bpf, x86: Swap 16-bit values with bswap and shr, not ror and movzx The 16-bit byte swap was emitted as a 16-bit rotate followed by a zero extending move. bswap of the 32-bit register followed by a right shift by 16 has the same result, is two bytes shorter, and avoids the operand size prefix and the 16-bit partial register write. Both sequences are two single cycle uops on Intel and AMD. LLVM emits be16 for bpf_htons() and bpf_ntohs() with cpu v3 and bswap16 with cpu v4, both of which take this path. Before, for r1 = be16 r1: 66 c1 cf 08 rorw $0x8,%di 0f b7 ff movzwl %di,%edi After: 0f cf bswap %edi c1 ef 10 shrl $0x10,%edi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-10-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1c84d2a02aba..50158d4b85b2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2596,18 +2596,16 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_END | BPF_FROM_LE: switch (imm32) { case 16: - /* Emit 'ror %ax, 8' to swap lower 2 bytes */ - EMIT1(0x66); + /* Emit 'bswap eax' to swap lower 4 bytes */ if (is_ereg(dst_reg)) EMIT1(0x41); - EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); + EMIT2(0x0F, add_1reg(0xC8, dst_reg)); - /* Emit 'movzwl eax, ax' */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + /* + * Emit 'shr eax, 16' to move the swapped lower + * 2 bytes back down and zero extend + */ + emit_shift_imm(&prog, BPF_RSH, dst_reg, 16, false); break; case 32: /* Emit 'bswap eax' to swap lower 4 bytes */ From f2a5c00a80f1457dce96397d7ce2c57968d130c5 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:22 +0200 Subject: [PATCH 496/660] bpf, x86: Use BMI2 shifts also when the count is in rcx Commit 77d8f5d47bfb ("bpf,x64: use shrx/sarx/shlx when available") kept the legacy shift by cl when the count already is in rcx, since that encoding is two bytes shorter. However, shl/shr/sar r,cl are 3 uops on Intel from Haswell through Skylake and still 2 uops on Ice Lake, as a zero count must leave the flags untouched, whereas shlx/shrx/sarx are a single uop with a latency of 1. On AMD both forms are a single uop, so the two extra bytes are the only cost there. Use the BMI2 form whenever it is available. LLVM keeps the count in whichever register it was allocated to, r4 included. Before/after for r1 <<= r4: 48 d3 e7 shlq %cl,%rdi c4 e2 f1 f7 ff shlxq %rcx,%rdi,%rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-11-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 50158d4b85b2..de49d7f51a54 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2539,8 +2539,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_X: - /* BMI2 shifts aren't better when shift count is already in rcx */ - if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) { + /* + * BMI2 shifts take the count from any register and, + * unlike the cl forms, are a single uop on Intel, so + * prefer them also when the count already is in rcx. + */ + if (boot_cpu_has(X86_FEATURE_BMI2)) { /* shrx/sarx/shlx dst_reg, dst_reg, src_reg */ bool w = (BPF_CLASS(insn->code) == BPF_ALU64); u8 op; From 74309a9294a74975fee75e4d6259eef026495325 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:23 +0200 Subject: [PATCH 497/660] bpf, x86: Fold the PROBE_MEM address computation into a lea The kernel address check in front of a BPF_PROBE_MEM load computed src_reg + off - VSYSCALL_ADDR with a mov of VSYSCALL_ADDR, a mov of the base register, an add of the offset in imm32 form and a sub, that is 20 bytes and 4 uops. VSYSCALL_ADDR is -10 MiB, so together with the 16-bit instruction offset it fits into a 32-bit displacement and a single lea does the same in 7 bytes and 1 uop on Intel and AMD. Before, for the check on r0 = *(u64 *)(r1 + 16): 49 c7 c2 00 00 60 ff movq $0xffffffffff600000,%r10 49 89 fb movq %rdi,%r11 49 81 c3 10 00 00 00 addq $0x10,%r11 4d 29 d3 subq %r10,%r11 After: 4c 8d 9f 10 00 a0 00 leaq 0xa00010(%rdi),%r11 Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-12-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de49d7f51a54..75194f9caa5a 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2853,22 +2853,16 @@ populate_extable: u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; u8 *end_of_jmp; - /* movabsq r10, VSYSCALL_ADDR */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32, - (u32)(long)VSYSCALL_ADDR); - - /* mov src_reg, r11 */ - EMIT_mov(AUX_REG, src_reg); - - if (insn->off) { - /* add r11, insn->off */ - maybe_emit_1mod(&prog, AUX_REG, true); - EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off); - } - - /* sub r11, r10 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); + /* + * lea r11, [src_reg + insn->off - VSYSCALL_ADDR] + * + * VSYSCALL_ADDR is a small negative number, so the + * whole offset fits into the 32-bit displacement. + */ + BUILD_BUG_ON((long)(s32)VSYSCALL_ADDR != (long)VSYSCALL_ADDR); + EMIT2(add_2mod(0x48, src_reg, AUX_REG), 0x8D); + emit_insn_suffix(&prog, src_reg, AUX_REG, + insn->off - (s32)VSYSCALL_ADDR); /* movabsq r10, limit */ emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, From 181576ebd87d90aafeedbecf7a2dcc109dcff391 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:24 +0200 Subject: [PATCH 498/660] bpf, x86: Compare the PROBE_MEM address range with a shift The PROBE_MEM check rejects src_reg + off - VSYSCALL_ADDR up to a limit of TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR, which does not fit into a 32-bit immediate and therefore needed a movabs into r10 and a register compare. Compare against the next power of two above the limit instead, which is a shift by fls64(limit) followed by a jne on the result. The addresses between the limit and that power of two are non-canonical on every x86-64 layout, since user space ends at TASK_SIZE_MAX and the kernel half starts far above 2^57, so they faulted and had the register zeroed by the exception handler before. Now they are zeroed by the check without taking the fault. The compare and branch stay two uops on Intel and AMD, while the sequence shrinks from 15 to 6 bytes. Before, for the compare of a 4-level paging kernel: 49 ba 00 00 a0 00 00 80 00 00 movabsq $0x800000a00000,%r10 4d 39 d3 cmpq %r10,%r11 77 05 ja 0x... After: 49 c1 eb 30 shrq $0x30,%r11 75 05 jne 0x... Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-13-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 75194f9caa5a..564cead6eab7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2848,6 +2848,11 @@ populate_extable: * src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE * and * src_reg + insn->off < VSYSCALL_ADDR + * + * The check is done on src_reg + insn->off - VSYSCALL_ADDR + * against the next power of two above the limit. Addresses + * between the limit and that power of two are non-canonical + * and would fault anyway, so they get zeroed a bit earlier. */ u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; @@ -2864,16 +2869,11 @@ populate_extable: emit_insn_suffix(&prog, src_reg, AUX_REG, insn->off - (s32)VSYSCALL_ADDR); - /* movabsq r10, limit */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, - (u32)(long)limit); + /* shr r11, fls64(limit) */ + emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true); - /* cmp r10, r11 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* if unsigned '>', goto load */ - EMIT2(X86_JA, 0); + /* if not zero, goto load */ + EMIT2(X86_JNE, 0); end_of_jmp = prog; /* xor dst_reg, dst_reg */ @@ -2881,7 +2881,7 @@ populate_extable: /* jmp byte_after_ldx */ EMIT2(0xEB, 0); - /* populate jmp_offset for JAE above to jump to start_of_ldx */ + /* populate jmp_offset for JNE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; } else if (!accesses_stack_only) { From 02cfd66f3d9a1aaab60b852f6bac902647db1e28 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:25 +0200 Subject: [PATCH 499/660] bpf, x86: Divide by constants via multiplication with a magic number LLVM strength-reduces 32-bit divisions by a constant into a 64-bit multiplication, but has no high-half multiply for the 64-bit ones, so these reach the JIT as BPF_DIV/BPF_MOD with BPF_K and were emitted as div or idiv, which take 20-100 cycles on Intel before Ice Lake, about 15 cycles on Ice Lake and later, and 9-19 cycles on AMD Zen 3-5, whereas a mul is 3 to 4 cycles on all current cores. Emit the multiplication with a magic number instead, with the constants computed per Hacker's Delight, 2nd ed., figures 10-1 and 10-2, which is also what compilers do for native code: an unsigned division becomes a mul plus a shift, or plus a sub/shr/add/shr sequence when the magic number needs 65 bits, which an even divisor avoids by shifting the dividend by its trailing zeros first, a signed one an imul, an optional add or sub of the dividend, a sar and the sign bit adjustment. A modulo multiplies the quotient back and subtracts it from the dividend. Divisors 0 and +-1 as well as unsigned divisors above 2^63 (negative imm32) are left to div/idiv as before, and so are the 32-bit divisions. The magic numbers were validated in user space against the bounds from Hacker's Delight for every divisor an imm32 can hold, in both signednesses, and the emitted sequences were executed with dividends around every multiple boundary of the divisor, at the extremes and at random. Before, for r1 /= 1000: 50 pushq %rax 52 pushq %rdx 49 c7 c3 e8 03 00 00 movq $0x3e8,%r11 48 89 f8 movq %rdi,%rax 31 d2 xorl %edx,%edx 49 f7 f3 divq %r11 48 89 c7 movq %rax,%rdi 5a popq %rdx 58 popq %rax After: 50 pushq %rax 52 pushq %rdx 48 89 f8 movq %rdi,%rax 48 c1 e8 03 shrq $0x3,%rax 49 bb cf f7 53 e3 a5 9b c4 20 movabsq $0x20c49ba5e353f7cf,%r11 49 f7 e3 mulq %r11 48 c1 ea 04 shrq $0x4,%rdx 48 89 d7 movq %rdx,%rdi 5a popq %rdx 58 popq %rax Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-14-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 219 +++++++++++++++++++++++++++++++++++- 1 file changed, 218 insertions(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 564cead6eab7..4e9c6bbe93d0 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1923,6 +1923,218 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) *pprog = prog; } +struct bpf_div_magic { + u64 m; /* multiplier, sign extended for signed division */ + int s; /* shift of the high half of the product */ + bool add; /* multiplier has 65 bits, use the add sequence */ +}; + +/* + * Multiplier and shift for an unsigned division by the constant d, with + * 2 <= d < 2^64 and a dividend below 2^(64 - lz): n / d == (mulhu(n, m) + * >> s), or with the add sequence when m needs 65 bits, (((n - t) >> 1) + * + t) >> (s - 1) with t = mulhu(n, m). Hacker's Delight, 2nd ed., figure + * 10-2. + */ +static void bpf_magic_udiv(u64 d, int lz, struct bpf_div_magic *mag) +{ + u64 mask = ~0ULL >> lz; + u64 nc, delta, q1, r1, q2, r2; + int p; + + mag->add = false; + /* Largest value with nc mod d == d - 1 */ + nc = mask - (mask % d + 1) % d; + p = 63; + q1 = (1ULL << 63) / nc; /* 2^p / nc */ + r1 = (1ULL << 63) - q1 * nc; /* 2^p mod nc */ + q2 = (~0ULL >> 1) / d; /* (2^p - 1) / d */ + r2 = (~0ULL >> 1) - q2 * d; /* (2^p - 1) mod d */ + do { + p++; + if (r1 >= nc - r1) { + q1 = 2 * q1 + 1; + r1 = 2 * r1 - nc; + } else { + q1 = 2 * q1; + r1 = 2 * r1; + } + if (r2 + 1 >= d - r2) { + if (q2 >= (~0ULL >> 1)) + mag->add = true; + q2 = 2 * q2 + 1; + r2 = 2 * r2 + 1 - d; + } else { + if (q2 >= (1ULL << 63)) + mag->add = true; + q2 = 2 * q2; + r2 = 2 * r2 + 1; + } + delta = d - 1 - r2; + } while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0))); + + mag->m = q2 + 1; + mag->s = p - 64; +} + +/* + * Multiplier and shift for a signed division by the constant d, with + * 2 <= |d| < 2^63: q = mulhs(n, m), q += n if d > 0 and m < 0, q -= n + * if d < 0 and m > 0, q >>= s (arithmetic), n / d == q + (q >>> 63). + * Hacker's Delight, 2nd ed., figure 10-1. + */ +static void bpf_magic_sdiv(s64 d, struct bpf_div_magic *mag) +{ + u64 ad = d < 0 ? -(u64)d : (u64)d; + u64 t, anc, delta, q1, r1, q2, r2; + int p; + + t = (1ULL << 63) + (d < 0); + anc = t - 1 - t % ad; /* |nc| */ + p = 63; + q1 = (1ULL << 63) / anc; /* 2^p / |nc| */ + r1 = (1ULL << 63) - q1 * anc; /* 2^p mod |nc| */ + q2 = (1ULL << 63) / ad; /* 2^p / |d| */ + r2 = (1ULL << 63) - q2 * ad; /* 2^p mod |d| */ + do { + p++; + q1 = 2 * q1; + r1 = 2 * r1; + if (r1 >= anc) { + q1++; + r1 -= anc; + } + q2 = 2 * q2; + r2 = 2 * r2; + if (r2 >= ad) { + q2++; + r2 -= ad; + } + delta = ad - r2; + } while (q1 < delta || (q1 == delta && r1 == 0)); + + mag->m = q2 + 1; + if (d < 0) + mag->m = -mag->m; + mag->s = p - 64; + mag->add = false; +} + +/* + * Divide dst_reg by the constant in insn->imm of a 64-bit division through + * a multiplication with the magic number from bpf_magic_udiv() or + * bpf_magic_sdiv(). The dividend goes through rax and the multiplier + * through r11, the caller saves rax and rdx unless they are dst_reg, and + * r10 is scratch. Returns false for the divisors left to div/idiv. + */ +static bool emit_div_const(u8 **pprog, const struct bpf_insn *insn, u32 dst_reg) +{ + bool is_mod = BPF_OP(insn->code) == BPF_MOD; + bool is_signed = insn->off == 1; + u32 n_reg = dst_reg, q_reg; + s32 imm32 = insn->imm; + struct bpf_div_magic mag; + u8 *prog = *pprog; + int corr = 0, pre = 0, s; + + if (is_signed) { + /* |d| >= 2, the verifier already rewrote d == -1 */ + if (imm32 >= -1 && imm32 <= 1) + return false; + bpf_magic_sdiv(imm32, &mag); + /* q += n for d > 0 and m < 0, q -= n for d < 0 and m > 0 */ + if (imm32 > 0 && (s64)mag.m < 0) + corr = 1; + else if (imm32 < 0 && (s64)mag.m > 0) + corr = -1; + } else { + /* d >= 2, a negative imm32 is a divisor above 2^63 */ + if (imm32 < 2) + return false; + bpf_magic_udiv(imm32, 0, &mag); + /* + * Shift the dividend by the trailing zeros of an even divisor + * first, the magic number of the odd part then fits without + * the add sequence. + */ + if (mag.add && !(imm32 & 1)) { + pre = __ffs(imm32); + bpf_magic_udiv(imm32 >> pre, pre, &mag); + if (mag.add) + return false; + } + } + s = mag.s; + /* + * The shift is at most 63, or 64 with the add sequence which shifts + * by s - 1 and needs s >= 1. Never hit for d >= 2, kept as a guard + * against an unencodable count. + */ + if (s > 64 - !mag.add || (mag.add && !s)) + return false; + + /* Keep the dividend when the multiplication clobbers it */ + if ((is_mod || mag.add || corr) && + (dst_reg == BPF_REG_0 || dst_reg == BPF_REG_3)) { + emit_mov_reg(&prog, true, BPF_REG_AX, dst_reg); + n_reg = BPF_REG_AX; + } + + /* mov rax, dst_reg; shr rax, pre; mov r11, m; mul r11 or imul r11 */ + if (dst_reg != BPF_REG_0) + emit_mov_reg(&prog, true, BPF_REG_0, dst_reg); + if (pre) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, pre, true); + emit_mov_imm64(&prog, AUX_REG, mag.m >> 32, (u32)mag.m); + EMIT3(0x49, 0xF7, is_signed ? 0xEB : 0xE3); + if (is_signed) { + /* add rdx, n_reg or sub rdx, n_reg */ + if (corr) + emit_alu_reg(&prog, corr > 0 ? BPF_ADD : BPF_SUB, + BPF_REG_3, n_reg, true); + /* sar rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_ARSH, BPF_REG_3, s, true); + /* mov rax, rdx; shr rax, 63; add rdx, rax */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_3); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 63, true); + emit_alu_reg(&prog, BPF_ADD, BPF_REG_3, BPF_REG_0, true); + q_reg = BPF_REG_3; + } else if (!mag.add) { + /* shr rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_3, s, true); + q_reg = BPF_REG_3; + } else { + /* mov rax, n_reg; sub rax, rdx; shr rax, 1 */ + emit_mov_reg(&prog, true, BPF_REG_0, n_reg); + emit_alu_reg(&prog, BPF_SUB, BPF_REG_0, BPF_REG_3, true); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 1, true); + /* add rax, rdx; shr rax, s - 1 */ + emit_alu_reg(&prog, BPF_ADD, BPF_REG_0, BPF_REG_3, true); + if (s > 1) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, s - 1, true); + q_reg = BPF_REG_0; + } + + if (is_mod) { + /* imul q_reg, q_reg, imm32 */ + maybe_emit_mod(&prog, q_reg, q_reg, true); + if (is_imm8(imm32)) + EMIT3(0x6B, add_2reg(0xC0, q_reg, q_reg), imm32); + else + EMIT2_off32(0x69, add_2reg(0xC0, q_reg, q_reg), imm32); + /* sub n_reg, q_reg */ + emit_alu_reg(&prog, BPF_SUB, n_reg, q_reg, true); + q_reg = n_reg; + } + if (q_reg != dst_reg) + emit_mov_reg(&prog, true, dst_reg, q_reg); + + *pprog = prog; + return true; +} + static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) { u8 *prog = *pprog; @@ -2443,6 +2655,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * if (dst_reg != BPF_REG_3) EMIT1(0x52); /* push rdx */ + /* LLVM strength-reduces the 32-bit ones itself */ + if (is64 && BPF_SRC(insn->code) == BPF_K && + emit_div_const(&prog, insn, dst_reg)) + goto div_done; + if (BPF_SRC(insn->code) == BPF_X) { if (src_reg == BPF_REG_0 || src_reg == BPF_REG_3) { @@ -2489,7 +2706,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * dst_reg != BPF_REG_0) /* mov dst_reg, rax */ emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); - +div_done: if (dst_reg != BPF_REG_3) EMIT1(0x5A); /* pop rdx */ if (dst_reg != BPF_REG_0) From 36b383f5abc39fc408a67957e40a9918220bbaf6 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:26 +0200 Subject: [PATCH 500/660] bpf, x86: Multiply by 3, 5 and 9 with lea LLVM turns a multiplication by a power of two into a shift, but keeps the other constants as a multiplication since BPF has no lea to lower them into, so r *= 3 and friends reach the JIT as BPF_MUL with BPF_K and were emitted as imul with an immediate, a 3 cycle instruction on Intel and AMD. Emit lea dst, [dst + dst * 2/4/8] for 3, 5 and 9 instead, which is the same size and a single cycle on Intel and on Zen 3 and later, two cycles on older AMD cores. r13 keeps the imul, since it needs a displacement byte as base register, which makes the lea a byte longer and a slow three operand form with 3 cycles of latency on Intel. Before/after for r1 *= 3: 48 6b ff 03 imulq $0x3,%rdi,%rdi 48 8d 3c 7f leaq (%rdi,%rdi,2),%rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-15-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 44 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 4e9c6bbe93d0..d1c22ae90fcc 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1163,17 +1163,22 @@ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) *pprog = prog; } -static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off) +/* + * Emit the suffix for addressing *(ptr_reg + (index_reg << scale) + off) + * and val_reg, with a SIB byte + */ +static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off, + u8 scale) { + u8 sib = (scale << 6) | add_2reg(0, ptr_reg, index_reg); u8 *prog = *pprog; if (!off && !base_needs_disp(ptr_reg)) { - EMIT2(add_2reg(0x04, BPF_REG_0, val_reg), - add_2reg(0, ptr_reg, index_reg) /* SIB */); + EMIT2(add_2reg(0x04, BPF_REG_0, val_reg), sib); } else if (is_imm8(off)) { - EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), sib, off); } else { - EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), sib, off); } *pprog = prog; } @@ -1444,7 +1449,7 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1466,7 +1471,7 @@ static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1537,7 +1542,7 @@ static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89); break; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; } @@ -1569,7 +1574,7 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7); break; } - emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off, 0); EMIT(imm, bpf_size_to_x86_bytes(size)); *pprog = prog; } @@ -1713,7 +1718,7 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); return -EFAULT; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; return 0; } @@ -2716,6 +2721,25 @@ div_done: case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: + /* + * lea for 3, 5 and 9, which LLVM leaves as a + * multiplication, but not for r13, which needs a + * displacement as base and makes the lea a slow three + * operand form on Intel. + */ + if ((imm32 == 3 || imm32 == 5 || imm32 == 9) && + !base_needs_disp(dst_reg)) { + /* lea dst_reg, [dst_reg + dst_reg * (imm32 - 1)] */ + if (BPF_CLASS(insn->code) == BPF_ALU64) + EMIT1(add_3mod(0x48, dst_reg, dst_reg, dst_reg)); + else if (is_ereg(dst_reg)) + EMIT1(add_3mod(0x40, dst_reg, dst_reg, dst_reg)); + EMIT1(0x8D); + emit_insn_suffix_SIB(&prog, dst_reg, dst_reg, dst_reg, 0, + ilog2(imm32 - 1)); + break; + } + maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); From 2378e471a6a2cd8e8baa7047f587781edb95975d Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:27 +0200 Subject: [PATCH 501/660] bpf, x86: Use emit_sub_rsp() for the trampoline stack The trampoline prologue open codes the choice between the imm8 and the imm32 form of sub rsp which emit_sub_rsp() already makes. Use it. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-16-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d1c22ae90fcc..c2fe72fa0bed 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -4325,13 +4325,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (im) im->ksym.fp_start = prog - (u8 *)rw_image; - if (!is_imm8(stack_size)) { - /* sub rsp, stack_size */ - EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - } else { - /* sub rsp, stack_size */ - EMIT4(0x48, 0x83, 0xEC, stack_size); - } + /* sub rsp, stack_size */ + emit_sub_rsp(&prog, stack_size); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) EMIT1(0x50); /* push rax */ /* mov QWORD PTR [rbp - rbx_off], rbx */ From b878ba01b67adccd637aa627bab78edd68a2f7dd Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:28 +0200 Subject: [PATCH 502/660] bpf, x86: Shorten the cast to a user address The cast of an arena pointer to its user address zero extended the low half, shifted it up, or'ed the user base in, rotated it back down and replaced the result with zero from a cleared r11 through a cmove for a NULL pointer: seven instructions with a dependency chain of six. Or the user base into r11 instead and pick it with a cmovne when the low half is not zero, five instructions with a chain of three. LLVM emits the cast for every arena pointer handed back to user space. Before, for r1 = addr_space_cast(r7, 1, 0) with a user base of 0x7f1200000000: 44 89 ef movl %r13d,%edi 48 c1 e7 20 shlq $0x20,%rdi 48 81 cf 12 7f 00 00 orq $0x7f12,%rdi 48 c1 c7 20 rolq $0x20,%rdi 4d 31 db xorq %r11,%r11 85 ff testl %edi,%edi 49 0f 44 fb cmoveq %r11,%rdi After: 44 89 ef movl %r13d,%edi 49 bb 00 00 00 00 12 7f 00 00 movabsq $0x7f1200000000,%r11 49 09 fb orq %rdi,%r11 85 ff testl %edi,%edi 49 0f 45 fb cmovneq %r11,%rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-17-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c2fe72fa0bed..3975a94ac812 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2544,35 +2544,18 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_MOV | BPF_X: if (insn_is_cast_user(insn)) { - if (dst_reg != src_reg) - /* 32-bit mov */ - emit_mov_reg(&prog, false, dst_reg, src_reg); - /* shl dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32); - - /* or dst_reg, user_vm_start */ - maybe_emit_1mod(&prog, dst_reg, true); - if (is_axreg(dst_reg)) - EMIT1_off32(0x0D, user_vm_start >> 32); - else - EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32); - - /* rol dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32); - - /* xor r11, r11 */ - EMIT3(0x4D, 0x31, 0xDB); - - /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */ + /* mov dst_reg32, src_reg32 to zero extend the low half */ + emit_mov_reg(&prog, false, dst_reg, src_reg); + /* mov r11, the high half of user_vm_start; or r11, dst_reg */ + emit_mov_imm64(&prog, AUX_REG, user_vm_start >> 32, 0); + emit_alu_reg(&prog, BPF_OR, AUX_REG, dst_reg, true); + /* test dst_reg32, dst_reg32; a NULL pointer stays NULL */ maybe_emit_mod(&prog, dst_reg, dst_reg, false); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); - - /* cmove r11, dst_reg; if so, set dst_reg to zero */ + /* cmovne dst_reg, r11 */ /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */ maybe_emit_mod(&prog, AUX_REG, dst_reg, true); - EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg)); + EMIT3(0x0F, 0x45, add_2reg(0xC0, AUX_REG, dst_reg)); break; } else if (insn_is_mov_percpu_addr(insn)) { /* mov , (if necessary) */ From 54e8bde13a3a089e06fea299e4b842aaf297a378 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:29 +0200 Subject: [PATCH 503/660] selftests/bpf: Add JIT instruction tests Carry only the relevant instruction tests of the test_bpf kernel module over into test_progs, so that BPF CI runs them on every JIT: ALU, atomic, 64-bit immediate load and jump instructions for the power-of-two magnitudes and signs of the operands, all shift counts, all operand register combinations, the division and modulo by constants, jumps with staggered offsets, here in both directions and also for the 32-bit unconditional jump, and the maximal jump over instructions which expand in the JITs. Add sweeps for the instruction patterns a JIT has a shorter or faster native form for, with every register as operand. The families with an immediate run once more with constant blinding. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t jit_insns [...] #191/1 jit_insns/arena access with zero offset:OK #191/2 jit_insns/cast to a user address:OK #191/3 jit_insns/alu_imm:OK #191/4 jit_insns/alu_reg:OK #191/5 jit_insns/alu_imm_regs:OK #191/6 jit_insns/div_const:OK #191/7 jit_insns/alu_reg_pairs:OK #191/8 jit_insns/shifts:OK #191/9 jit_insns/atomics:OK #191/10 jit_insns/atomic_reg_pairs:OK #191/11 jit_insns/xadd_chain:OK #191/12 jit_insns/ld_imm64:OK #191/13 jit_insns/jmp_imm:OK #191/14 jit_insns/jmp_reg:OK #191/15 jit_insns/jmp_reg_pairs:OK #191/16 jit_insns/staggered_jumps:OK #191/17 jit_insns/max_jmp:OK #191/18 jit_insns/ldst_zero_off:OK #191/19 jit_insns/ldst_offsets:OK #191/20 jit_insns/acq_rel:OK #191/21 jit_insns/movsx:OK #191/22 jit_insns/jset_imm:OK #191/23 jit_insns/cmp_imm:OK #191/24 jit_insns/alu_consts:OK #191/25 jit_insns/bswap:OK #191/26 jit_insns/probe_mem:OK #191 jit_insns:OK #192/1 jit_insns_blinded/alu_imm:OK #192/2 jit_insns_blinded/alu_imm_regs:OK #192/3 jit_insns_blinded/div_const:OK #192/4 jit_insns_blinded/shifts:OK #192/5 jit_insns_blinded/xadd_chain:OK #192/6 jit_insns_blinded/ld_imm64:OK #192/7 jit_insns_blinded/jmp_imm:OK #192/8 jit_insns_blinded/ldst_zero_off:OK #192/9 jit_insns_blinded/ldst_offsets:OK #192/10 jit_insns_blinded/jset_imm:OK #192/11 jit_insns_blinded/cmp_imm:OK #192/12 jit_insns_blinded/alu_consts:OK #192 jit_insns_blinded:OK Summary: 2/38 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-18-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/jit_insns.c | 2433 +++++++++++++++++ .../selftests/bpf/progs/jit_insns_arena.c | 113 + 2 files changed, 2546 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/jit_insns.c create mode 100644 tools/testing/selftests/bpf/progs/jit_insns_arena.c diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c new file mode 100644 index 000000000000..dabf8a4abdfe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -0,0 +1,2433 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include + +#include "sysctl_helpers.h" +#include "jit_insns_arena.skel.h" + +#define NR_CASES 512 +#define CTX_SPILL -16 /* stack slot holding the ctx pointer */ +#define MEM_SLOT -8 /* stack slot for the atomic operations */ + +struct sweep_ctx { + __u64 in[NR_CASES][2]; + __u64 out[NR_CASES][3]; +}; + +#define IN_OFF(k, j) (offsetof(struct sweep_ctx, in) + ((k) * 2 + (j)) * 8) +#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 3 + (j)) * 8) + +/* The cases of one program: operands in, results out, expected results */ +struct batch { + struct bpf_insn *insns; + int cnt, cap; + struct sweep_ctx ctx; + __u64 exp[NR_CASES][3]; + __u8 nr_out[NR_CASES]; + int nr; + char desc[80]; +}; + +static char log[1 << 16]; + +#define EMIT(b, insn) ((b)->insns[(b)->cnt++] = (insn)) + +static void emit_ld_imm64(struct batch *b, int reg, __u64 imm) +{ + EMIT(b, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(b, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + +static void batch_reset(struct batch *b) +{ + b->cnt = 0; + b->nr = 0; + /* The ctx pointer lives on the stack, every register is an operand */ + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, CTX_SPILL)); +} + +static bool batch_init(struct batch *b) +{ + b->cap = NR_CASES * 24 + 8; + b->insns = malloc(b->cap * sizeof(*b->insns)); + if (!ASSERT_OK_PTR(b->insns, "insns")) + return false; + batch_reset(b); + return true; +} + +static void batch_fini(struct batch *b) +{ + free(b->insns); +} + +static int load_prog(struct bpf_insn *insns, int cnt, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +/* Run the batch, check the results, start a new one */ +static void batch_flush(struct batch *b) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &b->ctx, + .ctx_size_in = sizeof(b->ctx), + ); + char buf[160]; + int fd, k, j; + + if (!b->nr) + return; + EMIT(b, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(b, BPF_EXIT_INSN()); + snprintf(buf, sizeof(buf), "%s: load %d cases", b->desc, b->nr); + fd = load_prog(b->insns, b->cnt, buf); + if (fd >= 0 && ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), b->desc)) { + for (k = 0; k < b->nr; k++) { + for (j = 0; j < b->nr_out[k]; j++) { + if (b->ctx.out[k][j] == b->exp[k][j]) + continue; + snprintf(buf, sizeof(buf), "%s: in0=0x%llx in1=0x%llx out%d", + b->desc, b->ctx.in[k][0], b->ctx.in[k][1], j); + ASSERT_EQ(b->ctx.out[k][j], b->exp[k][j], buf); + } + } + } + if (fd >= 0) + close(fd); + batch_reset(b); +} + +/* Add a case: record the operands and load the ctx pointer into rc */ +static int batch_case(struct batch *b, int rc, __u64 in0, __u64 in1) +{ + int k; + + if (b->nr == NR_CASES) + batch_flush(b); + k = b->nr++; + b->ctx.in[k][0] = in0; + b->ctx.in[k][1] = in1; + b->nr_out[k] = 0; + EMIT(b, BPF_LDX_MEM(BPF_DW, rc, BPF_REG_10, CTX_SPILL)); + return k; +} + +static void batch_expect(struct batch *b, int k, int j, __u64 val) +{ + b->exp[k][j] = val; + if (b->nr_out[k] <= j) + b->nr_out[k] = j + 1; +} + +#define LD_IN(b, k, j, rc, reg) EMIT(b, BPF_LDX_MEM(BPF_DW, reg, rc, IN_OFF(k, j))) +#define ST_OUT(b, k, j, rc, reg) EMIT(b, BPF_STX_MEM(BPF_DW, rc, reg, OUT_OFF(k, j))) + +/* A register that is neither of the two operand registers nor r0 */ +static int scratch_reg(int r1, int r2) +{ + int r; + + for (r = BPF_REG_1; r <= BPF_REG_9; r++) + if (r != r1 && r != r2) + return r; + return BPF_REG_9; +} + +static const char *alu_name(int op) +{ + switch (BPF_OP(op)) { + case BPF_MOV: return "mov"; + case BPF_NEG: return "neg"; + case BPF_AND: return "and"; + case BPF_OR: return "or"; + case BPF_XOR: return "xor"; + case BPF_LSH: return "lsh"; + case BPF_RSH: return "rsh"; + case BPF_ARSH: return "arsh"; + case BPF_ADD: return "add"; + case BPF_SUB: return "sub"; + case BPF_MUL: return "mul"; + case BPF_DIV: return "div"; + case BPF_MOD: return "mod"; + case BPF_OP(BPF_XCHG): return "xchg"; + case BPF_OP(BPF_CMPXCHG): return "cmpxchg"; + default: return "?"; + } +} + +static const char *jmp_name(int op) +{ + switch (op) { + case BPF_JSET: return "jset"; + case BPF_JEQ: return "jeq"; + case BPF_JNE: return "jne"; + case BPF_JGT: return "jgt"; + case BPF_JGE: return "jge"; + case BPF_JLT: return "jlt"; + case BPF_JLE: return "jle"; + case BPF_JSGT: return "jsgt"; + case BPF_JSGE: return "jsge"; + case BPF_JSLT: return "jslt"; + case BPF_JSLE: return "jsle"; + default: return "?"; + } +} + +/* ALU result computation, false when the operands are not valid */ +enum { F_ALU32 = 1, F_SIGNED = 2 }; + +static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) +{ + bool is_signed = flags & F_SIGNED; + + /* Narrow operands for ALU32 */ + if (flags & F_ALU32) { + if (is_signed) { + v1 = (__u64)(__s32)v1; + v2 = (__u64)(__s32)v2; + } else { + v1 = (__u32)v1; + v2 = (__u32)v2; + } + } + + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_NEG: + *res = -v1; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << v2; + break; + case BPF_RSH: + *res = v1 >> v2; + break; + case BPF_ARSH: + *res = v1 >> v2; + if (v2 > 0 && v1 > INT64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + case BPF_DIV: + if (v2 == 0) + return false; + if (!is_signed) { + *res = v1 / v2; + } else { + if ((__s64)v2 == -1) /* Handled by the verifier */ + return false; + *res = (__u64)((__s64)v1 / (__s64)v2); + } + break; + case BPF_MOD: + if (v2 == 0) + return false; + if (!is_signed) { + *res = v1 % v2; + } else { + if ((__s64)v2 == -1) + return false; + *res = (__u64)((__s64)v1 % (__s64)v2); + } + break; + } + return true; +} + +/* + * Operand pattern generator for the power-of-two magnitude tests: all + * combinations of magnitudes and signs with a block of contiguous values + * around each magnitude, with every dstep'th magnitude of the destination, + * and a larger block for the same magnitude of both operands. The first + * pattern is where the immediates take every encoding a JIT has, the + * second where the sign and overflow boundaries of both operands meet. + */ +static __s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 3 + +typedef void (*emit_fn)(struct batch *b, void *arg, __s64 dst, __s64 src); + +static void fill_pattern(struct batch *b, void *arg, int dbits, int sbits, + int block1, int dstep, int block2, emit_fn emit) +{ + static const int sgn[][2] = { { 1, 1 }, { 1, -1 }, { -1, 1 }, { -1, -1 } }; + int di, si, bt, db, sb, k; + + for (di = 0; di < dbits - 1; di += dstep) + for (si = 0; si < sbits - 1; si++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block1 / 2); db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); sb < (block1 + 1) / 2; sb++) + emit(b, arg, value(di, db, sgn[k][0]), + value(si, sb, sgn[k][1])); + + for (bt = 0; bt < (dbits > sbits ? dbits : sbits) - 1; bt++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); sb < (block2 + 1) / 2; sb++) + emit(b, arg, value(bt % dbits, db, sgn[k][0]), + value(bt % sbits, sb, sgn[k][1])); + batch_flush(b); +} + +/* ALU operations with an immediate and with a register for all magnitudes */ +static const int alu_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +struct alu_arg { + int op; + __u32 flags; +}; + +static void emit_alu_imm(struct batch *b, void *arg, __s64 dst, __s64 imm) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + if (!alu_result(&res, dst, (__s32)imm, a->op, a->flags)) + return; + k = batch_case(b, BPF_REG_3, dst, 0); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_K, + BPF_REG_1, 0, a->flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void emit_alu_reg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + if (!alu_result(&res, dst, src, a->op, a->flags)) + return; + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_X, + BPF_REG_1, BPF_REG_2, a->flags & F_SIGNED ? 1 : 0, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void test_alu_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_ops); o++) { + struct alu_arg a = { alu_ops[o], f ? F_ALU32 : 0 }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + for (o = 0; o < 2; o++) { + struct alu_arg a = { o ? BPF_MOD : BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0) }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_s%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + } + batch_fini(&b); +} + +static void test_alu_imm(void) +{ + test_alu_magnitudes(false); +} + +static void test_alu_reg(void) +{ + test_alu_magnitudes(true); +} + +/* ALU operations with an immediate for every destination register */ +static const int alu_reg_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_LSH, BPF_RSH, BPF_ARSH, + BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +static void alu_imm_regs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32; + __u32 imm = 0x01234567U; + __u64 dst, res; + int rd, rc, k; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + if (op == BPF_NEG) + imm = 0; + alu_result(&res, dst, imm, op, flags); + if (alu32) + res = (__u32)res; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k registers", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + k = batch_case(b, rc, dst, 0); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K, + rd, 0, flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, res); + } + batch_flush(b); +} + +static void test_alu_imm_regs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_imm_regs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_NEG, f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_imm_regs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* + * Division and modulo by constants for every width, signedness and + * destination register with a set of divisors and dividends around the + * multiple boundaries and the extremes of each range. This covers a JIT + * replacing the division with a multiplication by a magic number in all + * of its forms: magic numbers needing the extra add step, zero shifts, + * both signed corrections, and the divisors like 1, -1 and the ones + * above 2^63 that are still emitted as a real division. The dividends + * of the 32-bit operations carry garbage in the upper 32 bits. + */ +static const __s32 unsigned_divisors[] = { + 2, 3, 7, 10, 25, 641, 1000, 4096, 65536, 1000000007, 0x7fffffff, + -2, -10, -0x7fffffff - 1, -1, 1, +}; + +static const __s32 signed_divisors[] = { + 2, 3, 7, 25, 1000, 0x40000000, 0x7fffffff, + -2, -3, -5, -7, -1000, -0x7fffffff, -0x7fffffff - 1, 1, -1, +}; + +static int dividends(__u32 flags, __s32 imm, __u64 *n) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + __u64 d = is_signed ? (imm < 0 ? -(__s64)imm : imm) : + alu32 ? (__u32)imm : (__u64)(__s64)imm; + int cnt = 0, i; + + n[cnt++] = 0; + n[cnt++] = d - 1; + n[cnt++] = d + 1; + n[cnt++] = 3 * d + 2; + if (!alu32) { + n[cnt++] = (d << 40) - 1; + n[cnt++] = (d << 40) + 1; + n[cnt++] = (1ULL << 63) - 1; + n[cnt++] = 1ULL << 63; + n[cnt++] = ~0ULL; + n[cnt++] = 0xdaeb8ebd244a330cULL; + } else { + n[cnt++] = (d << 16) - 1; + n[cnt++] = (d << 16) + 1; + n[cnt++] = (1ULL << 31) - 1; + n[cnt++] = 1ULL << 31; + n[cnt++] = ~0U; + n[cnt++] = 0x8c1e86f1; + } + if (is_signed) { + n[cnt++] = -1; + n[cnt++] = -d - 1; + n[cnt++] = -3 * d - 2; + n[cnt++] = alu32 ? -(1ULL << 31) + 1 : -(1ULL << 63) + 1; + } + if (alu32) + for (i = 0; i < cnt; i++) + n[i] = (__u32)n[i] | ((0xa5a50000ULL + i) << 32); + return cnt; +} + +static __u64 div_result(int op, __u32 flags, __s32 imm, __u64 n) +{ + __u64 res; + + /* LLONG_MIN / -1 == LLONG_MIN, LLONG_MIN % -1 == 0, same for INT_MIN */ + if ((flags & F_SIGNED) && imm == -1) + res = op == BPF_MOD ? 0 : -n; + else + alu_result(&res, n, (__s32)imm, op, flags); + return flags & F_ALU32 ? (__u32)res : res; +} + +static void div_const(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + const __s32 *divs = is_signed ? signed_divisors : unsigned_divisors; + int cnt = is_signed ? ARRAY_SIZE(signed_divisors) : ARRAY_SIZE(unsigned_divisors); + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K; + int i, j, nr, rd, rc, k; + __u64 n[16]; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k constants", alu32 ? 32 : 64, + is_signed ? "s" : "", alu_name(op)); + for (i = 0; i < cnt; i++) { + nr = dividends(flags, divs[i], n); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + for (j = 0; j < nr; j++) { + k = batch_case(b, rc, n[j], divs[i]); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN(code, rd, 0, is_signed, divs[i])); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, div_result(op, flags, divs[i], n[j])); + } + } + } + batch_flush(b); +} + +static void test_div_const(void) +{ + struct batch b; + int f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + div_const(&b, BPF_DIV, f ? F_ALU32 : 0); + div_const(&b, BPF_MOD, f ? F_ALU32 : 0); + div_const(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + div_const(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* ALU operations for all combinations of operand registers */ +static void alu_reg_pairs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32; + __u64 dst, src, res, same; + int rd, rs, rc, k; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + src = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + src &= 31; + alu_result(&res, dst, src, op, flags); + alu_result(&same, src, src, op, flags); + if (alu32) { + res = (__u32)res; + same = (__u32)same; + } + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_x register pairs", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, rd == rs ? src : dst, src); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X, + rd, rs, flags & F_SIGNED ? 1 : 0, 0)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, rd == rs ? same : res); + } + } + batch_flush(b); +} + +static void test_alu_reg_pairs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_reg_pairs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_reg_pairs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_reg_pairs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* Shifts for all shift counts, by immediate, by register and by the shifted register */ +static void test_shifts(void) +{ + static const __s64 vals[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dword < 0, word > 0 */ + 0xfedcba0198765432LL, /* dword < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ + }; + static const int ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH }; + struct batch b; + int o, f, v, imm, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(ops); o++) { + int op = ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s all shift counts", bits, + alu_name(op)); + for (v = 0; v < ARRAY_SIZE(vals); v++) { + for (imm = 0; imm < bits; imm++) { + __u64 reg = vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, imm, op, 0); + if (f) + res = (__u32)res; + + k = batch_case(&b, BPF_REG_3, vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_IMM(op, BPF_REG_1, imm)); + else + EMIT(&b, BPF_ALU64_IMM(op, BPF_REG_1, imm)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + + k = batch_case(&b, BPF_REG_3, vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + } + } + /* The count in the shifted register */ + for (imm = 0; imm < bits; imm++) { + __u64 res; + + alu_result(&res, imm, imm, op, 0); + k = batch_case(&b, BPF_REG_3, imm, 0); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_1)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_1)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* Atomic operations on the stack for all magnitudes of memory and source operand */ +static const int atomic_ops[] = { + BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, + BPF_ADD | BPF_FETCH, BPF_AND | BPF_FETCH, BPF_OR | BPF_FETCH, BPF_XOR | BPF_FETCH, + BPF_XCHG, +}; + +struct atomic_arg { + int op; + int size; +}; + +static void emit_atomic(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct atomic_arg *a = arg; + bool w = a->size == BPF_W; + __u64 keep = 0x0123456789abcdefULL, res, fetch; + int k; + + if (a->op == BPF_XCHG) + res = src; + else + alu_result(&res, w ? (__u32)dst : dst, w ? (__u32)src : src, BPF_OP(a->op), 0); + /* The source register gets the old value with fetch, keeps its own without */ + fetch = a->op & BPF_FETCH ? (w ? (__u32)dst : dst) : src; + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_STX_MEM(a->size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + emit_ld_imm64(b, BPF_REG_0, keep); + EMIT(b, BPF_ATOMIC_OP(a->size, a->op, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(a->size, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_2); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + batch_expect(b, k, 0, w ? (__u32)res : res); + batch_expect(b, k, 1, fetch); + batch_expect(b, k, 2, keep); +} + +static void emit_cmpxchg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + int size = *(int *)arg; + bool w = size == BPF_W; + __u64 old = w ? (__u32)dst : dst; + int k; + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + /* Mismatch: the memory keeps its value, r0 gets it */ + EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1)); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_0); + /* Match: the memory gets the source, r0 the old value */ + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(size, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + batch_expect(b, k, 0, old); + batch_expect(b, k, 1, w ? (__u32)src : src); + batch_expect(b, k, 2, old); +} + +static void test_atomics(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) { + struct atomic_arg a = { atomic_ops[o], size }; + + snprintf(b.desc, sizeof(b.desc), "atomic%d_%s%s magnitudes", + size == BPF_DW ? 64 : 32, alu_name(a.op), + a.op == BPF_XCHG ? "" : a.op & BPF_FETCH ? "_fetch" : ""); + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_atomic); + } + snprintf(b.desc, sizeof(b.desc), "atomic%d_cmpxchg magnitudes", + size == BPF_DW ? 64 : 32); + fill_pattern(&b, &size, 64, 64, 0, 1, PATTERN_BLOCK2, emit_cmpxchg); + } + batch_fini(&b); +} + +/* Atomic operations for all combinations of pointer and source register */ +static void atomic_reg_pairs(struct batch *b, int size, int op) +{ + bool w = size == BPF_W; + __u64 mem, upd; + int rd, rs, rc, k, n; + + if (w) { + mem = 0x01234567U; + upd = 0x76543210U; + } else { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } + + snprintf(b->desc, sizeof(b->desc), "atomic%d_%s%s register pairs", w ? 32 : 64, + alu_name(op), op == BPF_XCHG || op == BPF_CMPXCHG ? "" : + op & BPF_FETCH ? "_fetch" : ""); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + __u64 regs[10], old = mem, new, src, cmp; + + /* The source would be the pointer, r0 the compare value */ + if (rs == rd || (op == BPF_CMPXCHG && rd == BPF_REG_0)) + continue; + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, mem, upd); + /* Memory from in0, r0 inverted, source from in1, pointer */ + LD_IN(b, k, 0, rc, rs); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT)); + LD_IN(b, k, 0, rc, BPF_REG_0); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_REG(rd, BPF_REG_10)); + EMIT(b, BPF_ATOMIC_OP(size, op, rd, rs, MEM_SLOT)); + /* r0 holds the pointer when it is the destination */ + n = 0; + if (rd != BPF_REG_0) + ST_OUT(b, k, n++, rc, BPF_REG_0); + ST_OUT(b, k, n++, rc, rs); + EMIT(b, BPF_LDX_MEM(size, rs, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, n, rc, rs); + + /* Model with the register aliasing of r0 */ + regs[BPF_REG_0] = ~mem; + regs[rs] = upd; + src = w ? (__u32)regs[rs] : regs[rs]; + if (op == BPF_XCHG) { + new = src; + regs[rs] = old; + } else if (op == BPF_CMPXCHG) { + cmp = w ? (__u32)regs[BPF_REG_0] : regs[BPF_REG_0]; + new = cmp == old ? src : old; + regs[BPF_REG_0] = old; + } else { + alu_result(&new, old, src, BPF_OP(op), 0); + if (w) + new = (__u32)new; + if (op & BPF_FETCH) + regs[rs] = old; + } + n = 0; + if (rd != BPF_REG_0) + batch_expect(b, k, n++, regs[BPF_REG_0]); + batch_expect(b, k, n++, regs[rs]); + batch_expect(b, k, n, new); + } + } + batch_flush(b); +} + +static void test_atomic_reg_pairs(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) + atomic_reg_pairs(&b, sizes[s], atomic_ops[o]); + atomic_reg_pairs(&b, sizes[s], BPF_CMPXCHG); + } + batch_fini(&b); +} + +/* A maximal chain of atomic adds on one stack slot */ +static void test_xadd_chain(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_insn *insns; + int len = BPF_MAXINSNS, s, i, fd; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + insns[0] = BPF_MOV32_IMM(BPF_REG_0, 1); + insns[1] = BPF_ST_MEM(sizes[s], BPF_REG_10, -40, 42); + for (i = 2; i < len - 2; i++) + insns[i] = BPF_STX_XADD(sizes[s], BPF_REG_10, BPF_REG_0, -40); + insns[len - 2] = BPF_LDX_MEM(sizes[s], BPF_REG_0, BPF_REG_10, -40); + insns[len - 1] = BPF_EXIT_INSN(); + + fd = load_prog(insns, len, "xadd chain"); + if (fd < 0) + break; + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "xadd chain run")) + ASSERT_EQ(opts.retval, 42 + len - 4, "xadd chain result"); + close(fd); + } + free(insns); +} + +/* 64-bit immediate loads for all magnitudes and byte patterns */ +static void ld_imm64_case(struct batch *b, __u64 imm) +{ + int k = batch_case(b, BPF_REG_3, imm, 0); + + emit_ld_imm64(b, BPF_REG_1, imm); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, imm); +} + +static void test_ld_imm64(void) +{ + static const struct { + __u8 base1, mask1, base2, mask2; + } bytes[] = { + { 0, 0xff, 0xff, 0xff }, /* checker */ + { 1, 0x81, 0x80, 0x80 }, /* positive and negative */ + { 1, 0x81, 0, 0xff }, /* positive and zero */ + { 0x80, 0x80, 0, 0xff }, /* negative and zero */ + }; + int block = 64, bit, adj, sign, p, pattern, index; + struct batch b; + + if (!batch_init(&b)) + return; + snprintf(b.desc, sizeof(b.desc), "ld_imm64 magnitudes"); + for (bit = 0; bit <= 62; bit++) + for (adj = -block / 2; adj < block / 2; adj++) + for (sign = -1; sign <= 1; sign += 2) + ld_imm64_case(&b, sign * ((1LL << bit) + adj)); + batch_flush(&b); + + for (p = 0; p < ARRAY_SIZE(bytes); p++) { + __u32 rand = 1; + + snprintf(b.desc, sizeof(b.desc), "ld_imm64 byte pattern %d", p); + for (pattern = 0; pattern < 256; pattern++) { + __u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & (1 << index)) + byte = (bytes[p].base1 & bytes[p].mask1) | + (rand & ~bytes[p].mask1); + else + byte = (bytes[p].base2 & bytes[p].mask2) | + (rand & ~bytes[p].mask2); + imm = (imm << 8) | (__u8)byte; + } + rand = rand * 1664525 + 1013904223; + ld_imm64_case(&b, imm); + } + batch_flush(&b); + } + batch_fini(&b); +} + +/* Conditional jumps with an immediate and with a register for all magnitudes */ +static const int jmp_ops[] = { + BPF_JSET, BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, +}; + +static bool jmp_match(__s64 v1, __s64 v2, __u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (__u64)v1 > (__u64)v2; + case BPF_JGE: + return (__u64)v1 >= (__u64)v2; + case BPF_JLT: + return (__u64)v1 < (__u64)v2; + case BPF_JLE: + return (__u64)v1 <= (__u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +struct jmp_arg { + int op; + bool is32; + bool reg; +}; + +static void emit_jmp(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct jmp_arg *a = arg; + bool match; + int k; + + if (a->is32) + match = jmp_match((__s32)dst, (__s32)src, a->op); + else + match = jmp_match(dst, a->reg ? src : (__s32)src, a->op); + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + if (a->reg) + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 1)); + if (a->reg && a->is32) + EMIT(b, BPF_JMP32_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->reg) + EMIT(b, BPF_JMP_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->is32) + EMIT(b, BPF_JMP32_IMM(a->op, BPF_REG_1, src, 1)); + else + EMIT(b, BPF_JMP_IMM(a->op, BPF_REG_1, src, 1)); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_4); + batch_expect(b, k, 0, match); +} + +static void test_jmp_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + struct jmp_arg a = { jmp_ops[o], f, reg }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c magnitudes", f ? "32" : "", + jmp_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_jmp); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 8, PATTERN_BLOCK2, + emit_jmp); + } + } + batch_fini(&b); +} + +static void test_jmp_imm(void) +{ + test_jmp_magnitudes(false); +} + +static void test_jmp_reg(void) +{ + test_jmp_magnitudes(true); +} + +/* A conditional jump on two registers, with the operands swapped for n == 1 */ +static void jmp_reg_pair(struct batch *b, bool is32, int op, int rd, int rs, int n) +{ + const __u64 a = 0x7edcba9876543210ULL, c = 0x0123456789abcdefULL; + __u64 v1 = n ? c : a, v2 = n ? a : c; + int rc, rt, k; + bool match; + + /* Two scratch registers besides the operands */ + rc = scratch_reg(rd, rs); + for (rt = BPF_REG_1; rt < BPF_REG_9; rt++) + if (rt != rd && rt != rs && rt != rc) + break; + if (rd == rs) + v2 = v1; + match = is32 ? jmp_match((__s32)v1, (__s32)v2, op) : jmp_match(v1, v2, op); + + k = batch_case(b, rc, v1, v2); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_IMM(rt, 1)); + if (is32) + EMIT(b, BPF_JMP32_REG(op, rd, rs, 1)); + else + EMIT(b, BPF_JMP_REG(op, rd, rs, 1)); + EMIT(b, BPF_MOV64_IMM(rt, 0)); + ST_OUT(b, k, 0, rc, rt); + batch_expect(b, k, 0, match); +} + +/* Conditional jumps for all combinations of operand registers, taken and not */ +static void test_jmp_reg_pairs(void) +{ + struct batch b; + int o, f, rd, rs, n; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_x register pairs", + f ? "32" : "", jmp_name(jmp_ops[o])); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) + for (n = 0; n < 2; n++) + jmp_reg_pair(&b, f, jmp_ops[o], rd, rs, n); + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* The offset of a jump instruction, which the 32-bit unconditional jump keeps in imm */ +static void set_jmp_off(struct bpf_insn *insn, bool ja32, int off) +{ + if (ja32) + insn->imm = off; + else + insn->off = off; +} + +static struct bpf_insn jmp_insn(bool ja32) +{ + return ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0) : BPF_JMP_IMM(BPF_JA, 0, 0, 0); +} + +/* + * Taken jumps over regions of doubling length, forward or backward, up to + * the maximal offset of the jump: 16 bits, or 2^17 for the 32-bit + * unconditional jump. The not taken path of a conditional jump and the + * guard of an unconditional one lead to an exit with the region number, so + * the verifier sees every instruction reachable and a jump landing anywhere + * else than on its target fails the run. + */ +static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool backward, + const char *desc) +{ + bool ja32 = BPF_CLASS(jmp->code) == BPF_JMP32 && BPF_OP(jmp->code) == BPF_JA; + bool ja = BPF_OP(jmp->code) == BPF_JA; + struct { __u64 r1, r2; } ctx = { r1, r2 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int max = ja32 ? 1 << 17 : INT16_MAX; + int len = 128 + 2 * max, i = 0, n = 0, gap, fd; + struct bpf_insn *insns; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); + for (gap = backward ? 4 : 2; gap <= max; gap = gap < max / 2 ? gap * 2 : max) { + int k, a, t, x, g, j; + + if (backward) { + /* + * Over the region to its guard and jump, which comes back + * to the start of the region, from where it leads on to + * the next one. + */ + a = i++; + t = i; + for (k = 0; k < gap - 3; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + x = i++; + g = i; + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + j = i; + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, t - (j + 1)); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n); + insns[i++] = BPF_EXIT_INSN(); + insns[a] = jmp_insn(ja32); + set_jmp_off(&insns[a], ja32, g - (a + 1)); + insns[x] = jmp_insn(ja32); + set_jmp_off(&insns[x], ja32, i - (x + 1)); + } else { + /* An unconditional jump needs a way into its region */ + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, gap); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n); + for (k = 0; k < gap - 2; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + insns[i++] = BPF_EXIT_INSN(); + } + if (gap == max) + break; + } + insns[i++] = BPF_EXIT_INSN(); + if (!ASSERT_LE(i, len, "insn count")) + goto out; + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, 0, desc); + close(fd); + } +out: + free(insns); +} + +static void test_staggered_jumps(void) +{ + /* A representative set, with operands which make the jump taken */ + static const struct { + const char *desc; + struct bpf_insn jmp; + __s64 r1, r2; + } kinds[] = { + { "staggered ja", + BPF_JMP_IMM(BPF_JA, 0, 0, 0), 0, 1 }, + { "staggered gotol", + BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0), 0, 1 }, + { "staggered jmp_jeq_k", + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 1234, 0), 1234, 0 }, + { "staggered jmp_jne_x", + BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_2, 0), 4321, 1234 }, + { "staggered jmp_jset_k", + BPF_JMP_IMM(BPF_JSET, BPF_REG_1, 0x82, 0), 0x86, 0 }, + { "staggered jmp_jsgt_x", + BPF_JMP_REG(BPF_JSGT, BPF_REG_1, BPF_REG_2, 0), -1, -2 }, + { "staggered jmp32_jlt_k", + BPF_JMP32_IMM(BPF_JLT, BPF_REG_1, 0x80000000, 0), 1234, 0 }, + { "staggered jmp32_jsge_x", + BPF_JMP32_REG(BPF_JSGE, BPF_REG_1, BPF_REG_2, 0), -2, -2 }, + }; + char desc[64]; + int i, d; + + for (i = 0; i < ARRAY_SIZE(kinds); i++) { + for (d = 0; d < 2; d++) { + snprintf(desc, sizeof(desc), "%s %s", kinds[i].desc, + d ? "backward" : "forward"); + staggered_jumps(&kinds[i].jmp, kinds[i].r1, kinds[i].r2, d, desc); + } + } +} + +/* + * The pattern sweeps: one program per pattern and register choice, run + * once per input with the operands in the ctx and the result handed back + * through it. + */ +struct pat_ctx { + __u64 in0; + __u64 in1; + __u64 res; +}; + +#define IN0 offsetof(struct pat_ctx, in0) +#define IN1 offsetof(struct pat_ctx, in1) +#define RES offsetof(struct pat_ctx, res) + +/* r6 holds the ctx */ +static const int pat_regs[] = { + BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, + BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, +}; + +/* The second register of a pattern: the same one, the next and a far one */ +#define NR_PAIRS 3 +static const int pair_off[NR_PAIRS] = { 0, 1, 5 }; + +static int pair_reg(int i, int k) +{ + return pat_regs[(i + pair_off[k]) % ARRAY_SIZE(pat_regs)]; +} + +static const __u64 inputs[] = { + 0, 0x1122334455667788ULL, 0xfedcba9876543210ULL, 0x80, 0xffffffffffffff7fULL, +}; + +struct prog { + struct bpf_insn insns[16]; + int cnt; +}; + +static void prog_start(struct prog *p) +{ + p->cnt = 0; + EMIT(p, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); +} + +/* Hand the result in reg back and return */ +static void prog_end(struct prog *p, int reg) +{ + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_6, reg, RES)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); +} + +static int prog_load(struct prog *p, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static void prog_run(int fd, __u64 in0, __u64 in1, __u64 expected, const char *desc) +{ + struct pat_ctx ctx = { .in0 = in0, .in1 = in1 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + char buf[128]; + + snprintf(buf, sizeof(buf), "%s in0=0x%llx in1=0x%llx", desc, in0, in1); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), buf)) + ASSERT_EQ(ctx.res, expected, buf); +} + +static int size_bits(int size) +{ + switch (size) { + case BPF_B: + return 8; + case BPF_H: + return 16; + case BPF_W: + return 32; + default: + return 64; + } +} + +/* The first size bytes of v in memory, as a load of that size reads them */ +static __u64 head_bytes(__u64 v, int size) +{ + int bits = size_bits(size); + + if (bits == 64) + return v; +#if __BYTE_ORDER == __BIG_ENDIAN + return v >> (64 - bits); +#else + return v & ((1ULL << bits) - 1); +#endif +} + +/* v with its first size bytes in memory replaced by those of w */ +static __u64 head_merge(__u64 v, __u64 w, int size) +{ + int bits = size_bits(size); + __u64 mask; + + if (bits == 64) + return w; + mask = (1ULL << bits) - 1; +#if __BYTE_ORDER == __BIG_ENDIAN + return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits)); +#else + return (v & ~mask) | (w & mask); +#endif +} + +static void test_ldst_zero_off(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const int aops[] = { + BPF_ADD, BPF_ADD | BPF_FETCH, BPF_OR, BPF_XCHG, BPF_CMPXCHG, + }; + static const __s32 imms[] = { 0x5a, -1 }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load with a zero offset, also into the base */ + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(size, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "ldx size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + /* Sign extending load with a zero offset from the stack */ + if (size != BPF_DW) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, + rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "ldsx size=%d base=r%d dst=r%d", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + v = size == BPF_B ? (__s8)v : + size == BPF_H ? (__s16)v : (__s32)v; + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } + + if (rd == rb) + continue; + /* Store with a zero offset over in0 */ + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_STX_MEM(size, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + + /* Store an immediate with a zero offset into a zeroed stack slot */ + for (k = 0; k < ARRAY_SIZE(imms); k++) { + int rd = pair_reg(b, 1); + + prog_start(&p); + EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ST_MEM(size, rb, 0, imms[k])); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", + size, rb, imms[k]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, + head_merge(0, (__u64)(__s64)imms[k], size), desc); + close(fd); + } + + /* Atomics with a zero offset on a stack slot holding in0 */ + for (k = 0; k < ARRAY_SIZE(aops); k++) { + int rd = pair_reg(b, 1), op = aops[k]; + + if (size == BPF_B || size == BPF_H) + break; + /* cmpxchg compares with r0 */ + if (op == BPF_CMPXCHG && (rb == BPF_REG_0 || rd == BPF_REG_0)) + continue; + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + if (op == BPF_CMPXCHG) + EMIT(&p, BPF_LDX_MEM(size, BPF_REG_0, BPF_REG_10, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, op, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "atomic 0x%x size=%d base=r%d src=r%d", + op, size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + __u64 v = head_bytes(inputs[j], size); + /* The source is the low half, whichever the byte order */ + __u64 w = size == BPF_W ? (__u32)in1 : in1; + __u64 r = BPF_OP(op) == BPF_ADD ? v + w : + BPF_OP(op) == BPF_OR ? v | w : w; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], r, size), desc); + } + close(fd); + } + } + } +} + +/* + * Loads, stores and stores of an immediate with the offsets around the + * limits of the short displacement forms, negative, large and unaligned + * ones, with every register as base. The memory is an array map value with + * the base register pointing at its middle, the loads hand their results + * back through a second value and the stores are checked against a copy + * kept here. The unaligned offsets are left out where the verifier insists + * on aligned accesses. + */ +#define MEM_SIZE 65536 +#define MEM_BASE (MEM_SIZE / 2) +#define MEM_KEY 0 +#define RES_KEY 1 +#define SLOT_KEY -4 /* stack slots: map key, results and base pointer */ +#define SLOT_RES -16 +#define SLOT_BASE -24 +#define STORE_VAL 0x8899aabbccddeeffULL + +struct off_prog { + struct bpf_insn insns[128]; + int cnt; +}; + +/* The map holding the memory and the results, a copy of the memory and a buffer */ +struct off_mem { + int map_fd; + __u8 *mem, *exp, *res; +}; + +enum { K_LDX, K_LDSX, K_STX, K_ST }; + +/* The offsets for an access of size, aligned unless unaligned ones are ok */ +static int offsets_for(int size, bool unaligned, int *off) +{ + int n = size_bits(size) / 8, cnt = 0; + + off[cnt++] = n; + off[cnt++] = 128 - n; + off[cnt++] = 128; + off[cnt++] = 256 - n; + off[cnt++] = 256; + off[cnt++] = 4096; + off[cnt++] = MEM_BASE - n; + off[cnt++] = -n; + off[cnt++] = -128; + off[cnt++] = -128 - n; + off[cnt++] = -256; + off[cnt++] = -4096; + off[cnt++] = -MEM_BASE; + if (n > 1 && unaligned) { + off[cnt++] = 13; + off[cnt++] = 127; + off[cnt++] = -13; + off[cnt++] = -129; + } + return cnt; +} + +/* The bytes a store of size writes from the register value v */ +static void store_bytes(__u8 *dst, __u64 v, int size) +{ + int n = size_bits(size) / 8; + +#if __BYTE_ORDER == __BIG_ENDIAN + memcpy(dst, (__u8 *)&v + 8 - n, n); +#else + memcpy(dst, &v, n); +#endif +} + +/* The value a load of size reads from the bytes at src */ +static __u64 load_bytes(const __u8 *src, int size) +{ + __u64 v = 0; + + memcpy(&v, src, size_bits(size) / 8); + return head_bytes(v, size); +} + +/* A register that is neither of the two given ones */ +static int other_reg(int r1, int r2) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pat_regs); i++) + if (pat_regs[i] != r1 && pat_regs[i] != r2) + return pat_regs[i]; + return BPF_REG_9; +} + +/* Look up the value of key into r0, exit with 1 when the lookup fails */ +static void off_prog_lookup(struct off_prog *p, int map_fd, int key) +{ + EMIT(p, BPF_ST_MEM(BPF_W, BPF_REG_10, SLOT_KEY, key)); + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, BPF_PSEUDO_MAP_FD, 0, map_fd)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, 0)); + EMIT(p, BPF_MOV64_REG(BPF_REG_2, BPF_REG_10)); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, SLOT_KEY)); + EMIT(p, BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem)); + EMIT(p, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 1)); + EMIT(p, BPF_EXIT_INSN()); +} + +/* rb points at the middle of the memory, the results value is spilled */ +static void off_prog_start(struct off_prog *p, int map_fd, int rb) +{ + p->cnt = 0; + off_prog_lookup(p, map_fd, RES_KEY); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_RES)); + off_prog_lookup(p, map_fd, MEM_KEY); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, MEM_BASE)); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_BASE)); + if (rb != BPF_REG_0) + EMIT(p, BPF_MOV64_REG(rb, BPF_REG_0)); +} + +static int off_prog_load(struct off_prog *p, const char *desc, bool quiet) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (quiet) + return fd; + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static int off_prog_run(int fd, const char *desc) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(fd, &opts); + close(fd); + if (!ASSERT_OK(err, desc) || !ASSERT_EQ(opts.retval, 0, desc)) + return -1; + return 0; +} + +/* Whether the verifier accepts a halfword load from an odd offset */ +static bool unaligned_ok(int map_fd) +{ + struct off_prog p; + int fd; + + off_prog_start(&p, map_fd, BPF_REG_1); + EMIT(&p, BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 1)); + fd = off_prog_load(&p, "unaligned probe", true); + if (fd < 0) + return false; + close(fd); + return true; +} + +/* One program loading from all the offsets of size through rb into rd */ +static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd, + const int *off, int nr) +{ + __u32 mem_key = MEM_KEY, res_key = RES_KEY; + int rs = other_reg(rb, rd), fd, i; + __u64 *res = (__u64 *)m->res; + int map_fd = m->map_fd; + struct off_prog p; + char desc[64]; + + off_prog_start(&p, map_fd, rb); + for (i = 0; i < nr; i++) { + if (rd == rb) + EMIT(&p, BPF_LDX_MEM(BPF_DW, rb, BPF_REG_10, SLOT_BASE)); + if (kind == K_LDX) + EMIT(&p, BPF_LDX_MEM(size, rd, rb, off[i])); + else + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, rd, rb, off[i], 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, SLOT_RES)); + EMIT(&p, BPF_STX_MEM(BPF_DW, rs, rd, i * 8)); + } + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, m->mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &res_key, res), desc)) + return; + for (i = 0; i < nr; i++) { + __u64 v = load_bytes(m->mem + MEM_BASE + off[i], size); + + if (kind == K_LDSX) + v = size == BPF_B ? (__s8)v : size == BPF_H ? (__s16)v : (__s32)v; + if (res[i] != v) { + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d off=%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd, off[i]); + ASSERT_EQ(res[i], v, desc); + } + } +} + +/* One program storing rd, or the immediate imm when rd < 0, at all the offsets */ +static void store_offsets(struct off_mem *m, int size, int rb, int rd, __s32 imm, + const int *off, int nr) +{ + __u64 val = rd >= 0 ? STORE_VAL : (__u64)(__s64)imm; + __u8 *mem = m->mem, *exp = m->exp, *res = m->res; + __u32 mem_key = MEM_KEY; + int map_fd = m->map_fd; + char desc[64], buf[96]; + struct off_prog p; + int fd, i; + + off_prog_start(&p, map_fd, rb); + if (rd >= 0) { + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, rd, 0, 0, (__u32)STORE_VAL)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, STORE_VAL >> 32)); + } + memcpy(exp, mem, MEM_SIZE); + for (i = 0; i < nr; i++) { + if (rd >= 0) + EMIT(&p, BPF_STX_MEM(size, rb, rd, off[i])); + else + EMIT(&p, BPF_ST_MEM(size, rb, off[i], imm)); + store_bytes(exp + MEM_BASE + off[i], val, size); + } + if (rd >= 0) + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", size, rb, rd); + else + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", size, rb, imm); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &mem_key, res), desc)) + return; + for (i = 0; i < MEM_SIZE && res[i] == exp[i]; i++) + ; + if (i < MEM_SIZE) { + snprintf(buf, sizeof(buf), "%s: byte at offset %d", desc, i - MEM_BASE); + ASSERT_EQ(res[i], exp[i], buf); + } +} + +static void test_ldst_offsets(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const __s32 imms[] = { -1, 0x7fffffff, -0x7fffffff - 1 }; + int b, k, s, i, nr, off[32]; + struct off_mem m = {}; + bool unaligned; + + m.map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "ldst_mem", sizeof(__u32), MEM_SIZE, 2, NULL); + if (!ASSERT_GE(m.map_fd, 0, "ldst_mem map")) + return; + m.mem = malloc(MEM_SIZE); + m.exp = malloc(MEM_SIZE); + m.res = malloc(MEM_SIZE); + if (!ASSERT_OK_PTR(m.mem, "mem") || !ASSERT_OK_PTR(m.exp, "exp") || + !ASSERT_OK_PTR(m.res, "res")) + goto out; + for (i = 0; i < MEM_SIZE; i++) + m.mem[i] = (i * 7 + 3) ^ (i >> 8); + unaligned = unaligned_ok(m.map_fd); + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int rb = pat_regs[b], size = sizes[s]; + + nr = offsets_for(size, unaligned, off); + /* Loads and sign extending loads, also into the base */ + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + load_offsets(&m, K_LDX, size, rb, rd, off, nr); + if (size != BPF_DW) + load_offsets(&m, K_LDSX, size, rb, rd, off, nr); + } + /* Stores of a register and of an immediate */ + for (k = 1; k < NR_PAIRS; k++) + store_offsets(&m, size, rb, pair_reg(b, k), 0, off, nr); + for (k = 0; k < ARRAY_SIZE(imms); k++) + store_offsets(&m, size, rb, -1, imms[k], off, nr); + } + } +out: + free(m.mem); + free(m.exp); + free(m.res); + close(m.map_fd); +} + +static void test_movsx(void) +{ + static const struct { + bool is64; + int off; + } ops[] = { + { true, 8 }, { true, 16 }, { true, 32 }, { false, 8 }, { false, 16 }, + }; + struct prog p; + char desc[64]; + int s, k, o, j, fd; + + for (s = 0; s < ARRAY_SIZE(pat_regs); s++) { + int rs = pat_regs[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(s, k); + + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_6, IN0)); + if (ops[o].is64) + EMIT(&p, BPF_MOVSX64_REG(rd, rs, ops[o].off)); + else + EMIT(&p, BPF_MOVSX32_REG(rd, rs, ops[o].off)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "movsx%d off=%d dst=r%d src=r%d", + ops[o].is64 ? 64 : 32, ops[o].off, rd, rs); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = inputs[j]; + + if (ops[o].off == 8) + v = (__s8)v; + else if (ops[o].off == 16) + v = (__s16)v; + else + v = (__s32)v; + prog_run(fd, inputs[j], 0, + ops[o].is64 ? (__u64)v : (__u32)v, desc); + } + close(fd); + } + } + } +} + +/* Branch on dst and imm, with 1 in res when taken and 0 otherwise */ +static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm) +{ + int rd = pat_regs[d], rt = pair_reg(d, 1); + + prog_start(p); + EMIT(p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(p, BPF_MOV64_IMM(rt, 1)); + if (is64) + EMIT(p, BPF_JMP_IMM(op, rd, imm, 1)); + else + EMIT(p, BPF_JMP32_IMM(op, rd, imm, 1)); + EMIT(p, BPF_MOV64_IMM(rt, 0)); + prog_end(p, rt); +} + +static void test_jset_imm(void) +{ + static const __s32 masks[] = { + 0x1, 0x40, 0x80, 0xff, 0x100, 0x7fff, 0x7fffffff, -1, -0x7fffffff - 1, + }; + struct prog p; + char desc[64]; + int d, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (m = 0; m < ARRAY_SIZE(masks); m++) { + jmp_prog(&p, is64, BPF_JSET, d, masks[m]); + snprintf(desc, sizeof(desc), "jset%d dst=r%d mask=0x%x", + is64 ? 64 : 32, rd, masks[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j]; + bool taken; + + if (is64) + taken = v & (__u64)(__s64)masks[m]; + else + taken = (__u32)v & (__u32)masks[m]; + prog_run(fd, v, 0, taken, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static bool cmp_taken(int op, bool is64, __u64 v, __u64 c) +{ + __s64 sv = is64 ? (__s64)v : (__s32)v, sc = is64 ? (__s64)c : (__s32)c; + + switch (op) { + case BPF_JEQ: + return v == c; + case BPF_JNE: + return v != c; + case BPF_JGT: + return v > c; + case BPF_JGE: + return v >= c; + case BPF_JLT: + return v < c; + case BPF_JLE: + return v <= c; + case BPF_JSGT: + return sv > sc; + case BPF_JSGE: + return sv >= sc; + case BPF_JSLT: + return sv < sc; + default: + return sv <= sc; + } +} + +static void test_cmp_imm(void) +{ + static const __s32 imms[] = { + 1, -1, 0x7f, 0x80, -0x80, -0x81, 0x1000, 0x7fffffff, + }; + static const int ops[] = { + BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, + }; + struct prog p; + char desc[64]; + int d, o, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + for (m = 0; m < ARRAY_SIZE(imms); m++) { + jmp_prog(&p, is64, ops[o], d, imms[m]); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o], rd, imms[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)imms[m] : + (__u32)imms[m]; + __u64 taken = cmp_taken(ops[o], is64, v, c); + + prog_run(fd, inputs[j], 0, taken, desc); + } + close(fd); + } + } + if (is64) + break; + } + } +} + +static __u64 alu_model(int op, __u64 v, __u64 c) +{ + switch (op) { + case BPF_ADD: + return v + c; + case BPF_SUB: + return v - c; + case BPF_AND: + return v & c; + case BPF_OR: + return v | c; + case BPF_XOR: + return v ^ c; + default: + return v * c; + } +} + +static void test_alu_consts(void) +{ + static const struct { + int op; + __s32 imm; + } ops[] = { + { BPF_ADD, 1 }, { BPF_ADD, -1 }, { BPF_SUB, 1 }, { BPF_SUB, -1 }, + { BPF_ADD, 128 }, { BPF_SUB, 128 }, { BPF_ADD, -128 }, + { BPF_AND, 0xff }, { BPF_AND, 0xffff }, { BPF_AND, 0x7f }, + { BPF_AND, 0x3ff }, { BPF_AND, 0x7fffffff }, + { BPF_OR, 0xff }, { BPF_XOR, -1 }, { BPF_XOR, 0x80 }, + { BPF_MUL, 2 }, { BPF_MUL, 3 }, { BPF_MUL, 4 }, { BPF_MUL, 5 }, + { BPF_MUL, 7 }, { BPF_MUL, 8 }, { BPF_MUL, 9 }, { BPF_MUL, 16 }, + { BPF_MUL, 0x40000000 }, { BPF_MUL, -3 }, { BPF_MUL, -0x7fffffff - 1 }, + { BPF_MUL, 0 }, { BPF_MUL, 1 }, { BPF_MUL, -1 }, + }; + struct prog p; + char desc[64]; + int d, o, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (is64) + EMIT(&p, BPF_ALU64_IMM(ops[o].op, rd, ops[o].imm)); + else + EMIT(&p, BPF_ALU32_IMM(ops[o].op, rd, ops[o].imm)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "alu%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o].op, rd, ops[o].imm); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)ops[o].imm : + (__u32)ops[o].imm; + __u64 r = alu_model(ops[o].op, v, c); + + prog_run(fd, inputs[j], 0, is64 ? r : (__u32)r, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static void test_bswap(void) +{ + static const int widths[] = { 16, 32, 64 }; + struct prog p; + char desc[64]; + int d, w, o, j, fd; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (w = 0; w < ARRAY_SIZE(widths); w++) { + int bits = widths[w]; + + /* To big endian, to little endian, and the unconditional swap */ + for (o = 0; o < 3; o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (o == 0) + EMIT(&p, BPF_ENDIAN(BPF_TO_BE, rd, bits)); + else if (o == 1) + EMIT(&p, BPF_ENDIAN(BPF_TO_LE, rd, bits)); + else + EMIT(&p, BPF_RAW_INSN(BPF_ALU64 | BPF_END | BPF_TO_LE, + rd, 0, 0, bits)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "%s%d dst=r%d", + o == 0 ? "be" : o == 1 ? "le" : "bswap", bits, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j], r; + + if (bits == 16) + r = o == 0 ? htobe16(v) : o == 1 ? htole16(v) : + __builtin_bswap16(v); + else if (bits == 32) + r = o == 0 ? htobe32(v) : o == 1 ? htole32(v) : + __builtin_bswap32(v); + else + r = o == 0 ? htobe64(v) : o == 1 ? htole64(v) : + __builtin_bswap64(v); + prog_run(fd, v, 0, r, desc); + } + close(fd); + } + } + } +} + +static void test_probe_mem(void) +{ + int kfunc_id, type_id, b, k, j, fd; + struct btf *btf; + struct prog p; + char desc[64]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_rdonly_cast", BTF_KIND_FUNC); + type_id = btf__find_by_name_kind(btf, "timespec64", BTF_KIND_STRUCT); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_rdonly_cast id") || + !ASSERT_GT(type_id, 0, "timespec64 id")) + return; + + /* The kernel copy of the ctx through an untrusted pointer in every register */ + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, + 0, kfunc_id)); + if (rb != BPF_REG_0) + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "probe_mem base=r%d dst=r%d", rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, inputs[j], desc); + close(fd); + } + } + + /* NULL and a user address are not touched and read as zero */ + for (j = 0; j < 2; j++) { + __u64 addr = j ? 0x00007ffffffff000ULL : 0; + + prog_start(&p); + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, 0, 0, (__u32)addr)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, addr >> 32)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, + 0, kfunc_id)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0)); + prog_end(&p, BPF_REG_3); + snprintf(desc, sizeof(desc), "probe_mem addr=0x%llx", addr); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 0, desc); + close(fd); + } +} + +/* Whether the JIT supports load-acquire and store-release */ +static bool acq_rel_ok(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, .prog_flags = BPF_F_SLEEPABLE); + struct prog p; + int fd; + + prog_start(&p); + EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0)); + EMIT(&p, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)); + prog_end(&p, BPF_REG_0); + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p.insns, p.cnt, &lopts); + if (fd < 0) + return false; + close(fd); + return true; +} + +/* + * Load-acquire and store-release of every size with every register as + * base, on a stack slot holding in0: the loads zero extend like the plain + * loads and the stores replace the first bytes of the slot like the plain + * stores. + */ +static void test_acq_rel(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + if (!acq_rel_ok()) { + test__skip(); + return; + } + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load-acquire from the slot, also into the base */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_LOAD_ACQ, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "load_acq size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + if (rd == rb) + continue; + /* Store-release of in1 over the slot holding in0 */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_STORE_REL, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "store_rel size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + } + } +} + +/* + * A jump with the maximal offset over instructions which expand to several + * native ones, so that the native offset outgrows the field of a short + * branch and takes the JITs through their branch conversion. The region + * must not grow in the verifier, or the jump could not reach over it any + * more: the divisions are by constants, as a division by a register gets + * a zero check in front, and the region sticks to one width, as a 32-bit + * result read by a 64-bit instruction gets a zero extension on the JITs + * which need one. The region is the not taken path and ends with a result + * different from the taken one, so a jump landing inside it fails the run. + */ +static void max_jmp(bool alu32, bool taken) +{ + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + struct { __u64 r0, r2; } ctx = { taken, 0x0123456789abcdefULL }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int len = INT16_MAX + 5, i = 0, fd; + struct bpf_insn *insns; + char desc[64]; + + snprintf(desc, sizeof(desc), "max jump alu%d %s", alu32 ? 32 : 64, + taken ? "taken" : "not taken"); + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0); + insns[i++] = BPF_MOV64_REG(BPF_REG_3, BPF_REG_2); + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, INT16_MAX); + while (i < len - 2) { + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; + + if (op == BPF_DIV || op == BPF_MOD) + insns[i++] = alu32 ? BPF_ALU32_IMM(op, BPF_REG_3, 7) : + BPF_ALU64_IMM(op, BPF_REG_3, 7); + else + insns[i++] = alu32 ? BPF_ALU32_REG(op, BPF_REG_3, BPF_REG_2) : + BPF_ALU64_REG(op, BPF_REG_3, BPF_REG_2); + } + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 2); + insns[i++] = BPF_EXIT_INSN(); + ASSERT_EQ(i, len, "insn count"); + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, taken ? 1 : 2, desc); + close(fd); + } + free(insns); +} + +static void test_max_jmp(void) +{ + max_jmp(false, true); + max_jmp(false, false); + max_jmp(true, true); + max_jmp(true, false); +} + +void test_jit_insns(void) +{ + RUN_TESTS(jit_insns_arena); + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_reg")) + test_alu_reg(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("alu_reg_pairs")) + test_alu_reg_pairs(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("atomics")) + test_atomics(); + if (test__start_subtest("atomic_reg_pairs")) + test_atomic_reg_pairs(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("jmp_reg")) + test_jmp_reg(); + if (test__start_subtest("jmp_reg_pairs")) + test_jmp_reg_pairs(); + if (test__start_subtest("staggered_jumps")) + test_staggered_jumps(); + if (test__start_subtest("max_jmp")) + test_max_jmp(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("acq_rel")) + test_acq_rel(); + if (test__start_subtest("movsx")) + test_movsx(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + if (test__start_subtest("bswap")) + test_bswap(); + if (test__start_subtest("probe_mem")) + test_probe_mem(); +} + +/* The families with an immediate once more with the constants blinded */ +void serial_test_jit_insns_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + sysctl_set_or_fail(harden, NULL, old); +} diff --git a/tools/testing/selftests/bpf/progs/jit_insns_arena.c b/tools/testing/selftests/bpf/progs/jit_insns_arena.c new file mode 100644 index 000000000000..b396d5aefb5a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/jit_insns_arena.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The arena accesses of jit_insns in their own object, so that a kernel + * without arena support fails only these programs. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The first bytes in memory of the test value and its sum with the slot + * after the u32 store depend on the byte order. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define VAL_B0 0x88 +#define VAL_H0 0x7788 +#define VAL_W0 0x55667788 +#define VAL_SUM 0x22446688556689bc +#else +#define VAL_B0 0x11 +#define VAL_H0 0x1122 +#define VAL_W0 0x11223344 +#define VAL_SUM 0x11224578aaccef10 +#endif + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +SEC("syscall") +__description("arena access with zero offset") +__success __retval(0) +int arena_zero_off(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + res = (__u32)(__u64)page; + /* Accesses through two base registers */ + asm volatile (" \ + r7 = %[res]; \ + r7 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = r7; \ + r8 = 0x1122334455667788 ll; \ + *(u64 *)(r7 + 0) = r8; \ + r9 = *(u64 *)(r7 + 0); \ + r9 ^= r8; \ + *(u64 *)(r1 + 0) = r8; \ + r2 = *(u64 *)(r1 + 0); \ + r2 ^= r8; \ + r9 |= r2; \ + r3 = *(u8 *)(r1 + 0); \ + r3 ^= %[b0]; \ + r9 |= r3; \ + r4 = *(u16 *)(r1 + 0); \ + r4 ^= %[h0]; \ + r9 |= r4; \ + r5 = *(u32 *)(r1 + 0); \ + r5 ^= %[w0]; \ + r9 |= r5; \ + *(u32 *)(r1 + 0) = 0x1234; \ + lock *(u64 *)(r7 + 0) += r8; \ + r2 = *(u64 *)(r1 + 0); \ + r8 = %[sum] ll; \ + r2 ^= r8; \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"+r"(res) + : __imm_const(b0, VAL_B0), __imm_const(h0, VAL_H0), + __imm_const(w0, VAL_W0), __imm_const(sum, VAL_SUM) + : "r1", "r2", "r3", "r4", "r5", "r7", "r8", "r9"); + return res != 0; +} + +SEC("syscall") +__description("cast to a user address") +__success __retval(0) +int cast_to_user(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + /* The user address comes back from the kernel one, NULL stays NULL */ + asm volatile (" \ + r7 = %[page]; \ + r1 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = addr_space_cast(r1, 0x1, 0x0); \ + r9 = r1; \ + r9 ^= r7; \ + r2 = 0; \ + r2 = addr_space_cast(r2, 0x1, 0x0); \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"=r"(res) + : [page]"r"(page) + : "r1", "r2", "r7", "r9"); + return res != 0; +} +#endif + +char _license[] SEC("license") = "GPL"; From 4532e525b286328b33c2be592406464f4e671689 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:30 +0200 Subject: [PATCH 504/660] selftests/bpf: Remove the old test_bpf module The instruction tests of the test_bpf module live in the BPF selftests now, as the jit_insns test of test_progs, which BPF CI runs on every architecture it covers, and its skb_segment tests have a KUnit successor in net/core/net_test.c. Remove the module together with its Kconfig entry, the test_kmod.sh and test_bpf.sh wrappers of the bpf and net selftests, the config fragments and defconfigs that enabled it, as well as all other references to it and symbol exports in core kernel. For archs wanting to support BPF JITs, it's time they move over to the BPF selftests for proper coverage. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-19-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- Documentation/bpf/bpf_devel_QA.rst | 16 +- Documentation/networking/filter.rst | 15 +- MAINTAINERS | 1 - arch/m68k/configs/amiga_defconfig | 1 - arch/m68k/configs/apollo_defconfig | 1 - arch/m68k/configs/atari_defconfig | 1 - arch/m68k/configs/bvme6000_defconfig | 1 - arch/m68k/configs/hp300_defconfig | 1 - arch/m68k/configs/mac_defconfig | 1 - arch/m68k/configs/multi_defconfig | 1 - arch/m68k/configs/mvme147_defconfig | 1 - arch/m68k/configs/mvme16x_defconfig | 1 - arch/m68k/configs/q40_defconfig | 1 - arch/m68k/configs/sun3_defconfig | 1 - arch/m68k/configs/sun3x_defconfig | 1 - arch/powerpc/configs/ppc64_defconfig | 1 - arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - kernel/bpf/core.c | 4 - lib/Kconfig.debug | 13 - lib/Makefile | 1 - lib/test_bpf.c | 15934 --------------------- tools/bpf/bpftool/feature.c | 3 - tools/testing/selftests/bpf/Makefile | 3 +- tools/testing/selftests/bpf/config | 1 - tools/testing/selftests/bpf/test_kmod.sh | 73 - tools/testing/selftests/net/Makefile | 1 - tools/testing/selftests/net/config | 1 - tools/testing/selftests/net/test_bpf.sh | 11 - 29 files changed, 15 insertions(+), 16077 deletions(-) delete mode 100644 lib/test_bpf.c delete mode 100755 tools/testing/selftests/bpf/test_kmod.sh delete mode 100755 tools/testing/selftests/net/test_bpf.sh diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst index edf8107a7beb..4729efd657c3 100644 --- a/Documentation/bpf/bpf_devel_QA.rst +++ b/Documentation/bpf/bpf_devel_QA.rst @@ -374,9 +374,10 @@ developers in order to get the feature implemented in a timely manner. Please refer to the git log (``arch/*/net/``) to locate the necessary people for helping out. -Also always make sure to add BPF test cases (e.g. test_bpf.c and -test_verifier.c) for new instructions, so that they can receive -broad test coverage and help run-time testing the various BPF JITs. +Also always make sure to add BPF test cases (e.g. the jit_insns test +in test_progs and test_verifier.c) for new instructions, so that they +can receive broad test coverage and help run-time testing the various +BPF JITs. In case of new BPF instructions, once the changes have been accepted into the Linux kernel, please implement support into LLVM's BPF back @@ -514,10 +515,11 @@ A: If you run a kernel ``xyz``, then always run the BPF kernel selftests from that kernel ``xyz`` as well. Do not expect that the BPF selftest from the latest mainline tree will pass all the time. -In particular, test_bpf.c and test_verifier.c have a large number of -test cases and are constantly updated with new BPF test sequences, or -existing ones are adapted to verifier changes e.g. due to verifier -becoming smarter and being able to better track certain things. +In particular, test_verifier.c and the instruction tests in test_progs +have a large number of test cases and are constantly updated with new +BPF test sequences, or existing ones are adapted to verifier changes +e.g. due to verifier becoming smarter and being able to better track +certain things. LLVM ==== diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index 8eb9a5d40f31..e0b9dc14a9bf 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst @@ -656,16 +656,11 @@ from eBPF instruction set. Testing ------- -Next to the BPF toolchain, the kernel also ships a test module that contains -various test cases for classic and eBPF that can be executed against -the BPF interpreter and JIT compiler. It can be found in lib/test_bpf.c and -enabled via Kconfig:: - - CONFIG_TEST_BPF=m - -After the module has been built and installed, the test suite can be executed -via insmod or modprobe against 'test_bpf' module. Results of the test cases -including timings in nsec can be found in the kernel log (dmesg). +Next to the BPF toolchain, the BPF selftests under tools/testing/selftests/bpf +ship instruction tests in test_progs, see jit_insns, which runs the eBPF +instructions over a range of operand magnitudes and signs, register +combinations, memory offsets and jump offsets against the BPF interpreter +or the JIT compiler, depending on the net.core.bpf_jit_enable setting. Misc ---- diff --git a/MAINTAINERS b/MAINTAINERS index fe99b7437a2d..80845ba7dfb5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4981,7 +4981,6 @@ F: include/uapi/linux/filter.h F: kernel/bpf/ F: kernel/trace/bpf_trace.c F: lib/buildid.c -F: lib/test_bpf.c F: net/bpf/ F: net/core/filter.c F: net/sched/act_bpf.c diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 8b7efe20dd12..0efd78d64a90 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -576,7 +576,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index c7f7705806f9..35b54dc7e79c 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -531,7 +531,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 2fb213161bae..265588b71b82 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -554,7 +554,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 6730242d4d48..595601e22971 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -523,7 +523,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 2b3ff26ac8fd..568d80e3db22 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -533,7 +533,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 61f32ef2547e..da1f4677aa6d 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -552,7 +552,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 2cbd6bf20a68..8e6466c51484 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -639,7 +639,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 019f8b8070be..8e7882354e53 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -523,7 +523,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 389163287425..d3e7049c2e43 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -524,7 +524,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index cbc41443cffa..c517c32dbdc7 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -542,7 +542,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 273e840dc8d6..924f5f05906f 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -520,7 +520,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 843d72924bcb..f63c6392d92e 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -521,7 +521,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 1eb8e3457e8b..e0b7901add11 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -442,7 +442,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_USER_COPY=m -CONFIG_TEST_BPF=m CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 3dae71474333..d597b4b532a3 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -927,6 +927,5 @@ CONFIG_INTERVAL_TREE_TEST=m CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y CONFIG_TEST_BITOPS=m -CONFIG_TEST_BPF=m CONFIG_PAGE_TABLE_CHECK=y CONFIG_PAGE_TABLE_CHECK_ENFORCED=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 6f5722634b4d..c417b1717e89 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -851,4 +851,3 @@ CONFIG_LKDTM=m CONFIG_KPROBES_SANITY_TEST=m CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y -CONFIG_TEST_BPF=m diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 05c89396119a..642f0ab8eb2d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -172,7 +172,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) } return prog; } -EXPORT_SYMBOL_GPL(bpf_prog_alloc); int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { @@ -1681,7 +1680,6 @@ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) { return 0; } -EXPORT_SYMBOL_GPL(__bpf_call_base); /* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \ @@ -2747,7 +2745,6 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) { return __bpf_prog_select_runtime(NULL, fp, err); } -EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); static unsigned int __bpf_prog_ret1(const void *ctx, const struct bpf_insn *insn) @@ -3138,7 +3135,6 @@ void bpf_prog_free(struct bpf_prog *fp) INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } -EXPORT_SYMBOL_GPL(bpf_prog_free); /* RNG for unprivileged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2f34619905b6..d3d3604bedcf 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2638,19 +2638,6 @@ config TEST_WORKQUEUE If unsure, say N. -config TEST_BPF - tristate "Test BPF filter functionality" - depends on m && NET - help - This builds the "test_bpf" module that runs various test vectors - against the BPF interpreter or BPF JIT compiler depending on the - current setting. This is in particular useful for BPF JIT compiler - development, but also to run regression tests against changes in - the interpreter code. It also enables test stubs for eBPF maps and - verifier used by user space verifier testsuite. - - If unsure, say N. - config FIND_BIT_BENCHMARK tristate "Test find_bit functions" help diff --git a/lib/Makefile b/lib/Makefile index dfab958327c5..361b02903bef 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -66,7 +66,6 @@ obj-y += kstrtox.o obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o obj-$(CONFIG_REGION_ALLOC_BENCHMARK) += region_alloc_benchmark.o obj-$(CONFIG_FIND_BIT_BENCHMARK_RUST) += find_bit_benchmark_rust.o -obj-$(CONFIG_TEST_BPF) += test_bpf.o test_dhry-objs := dhry_1.o dhry_2.o dhry_run.o obj-$(CONFIG_TEST_DHRY) += test_dhry.o obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o diff --git a/lib/test_bpf.c b/lib/test_bpf.c deleted file mode 100644 index af6f3340c034..000000000000 --- a/lib/test_bpf.c +++ /dev/null @@ -1,15934 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Testsuite for BPF interpreter and BPF JIT compiler - * - * Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* General test specific settings */ -#define MAX_SUBTESTS 3 -#define MAX_TESTRUNS 1000 -#define MAX_DATA 128 -#define MAX_INSNS 512 -#define MAX_K 0xffffFFFF - -/* Few constants used to init test 'skb' */ -#define SKB_TYPE 3 -#define SKB_MARK 0x1234aaaa -#define SKB_HASH 0x1234aaab -#define SKB_QUEUE_MAP 123 -#define SKB_VLAN_TCI 0xffff -#define SKB_VLAN_PRESENT 1 -#define SKB_DEV_IFINDEX 577 -#define SKB_DEV_TYPE 588 - -/* Redefine REGs to make tests less verbose */ -#define R0 BPF_REG_0 -#define R1 BPF_REG_1 -#define R2 BPF_REG_2 -#define R3 BPF_REG_3 -#define R4 BPF_REG_4 -#define R5 BPF_REG_5 -#define R6 BPF_REG_6 -#define R7 BPF_REG_7 -#define R8 BPF_REG_8 -#define R9 BPF_REG_9 -#define R10 BPF_REG_10 - -/* Flags that can be passed to test cases */ -#define FLAG_NO_DATA BIT(0) -#define FLAG_EXPECTED_FAIL BIT(1) -#define FLAG_SKB_FRAG BIT(2) -#define FLAG_VERIFIER_ZEXT BIT(3) -#define FLAG_LARGE_MEM BIT(4) - -enum { - CLASSIC = BIT(6), /* Old BPF instructions only. */ - INTERNAL = BIT(7), /* Extended instruction set. */ -}; - -#define TEST_TYPE_MASK (CLASSIC | INTERNAL) - -struct bpf_test { - const char *descr; - union { - struct sock_filter insns[MAX_INSNS]; - struct bpf_insn insns_int[MAX_INSNS]; - struct { - void *insns; - unsigned int len; - } ptr; - } u; - __u8 aux; - __u8 data[MAX_DATA]; - struct { - int data_size; - __u32 result; - } test[MAX_SUBTESTS]; - int (*fill_helper)(struct bpf_test *self); - int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ - __u8 frag_data[MAX_DATA]; - int stack_depth; /* for eBPF only, since tests don't call verifier */ - int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ -}; - -/* Large test cases need separate allocation and fill handler. */ - -static int bpf_fill_maxinsns1(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - __u32 k = ~0; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++, k--) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, k); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns2(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns3(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - struct rnd_state rnd; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - prandom_seed_state(&rnd, 3141592653589793238ULL); - - for (i = 0; i < len - 1; i++) { - __u32 k = prandom_u32_state(&rnd); - - insn[i] = __BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, k); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns4(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS + 1; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns5(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns6(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_VLAN_TAG_PRESENT); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns7(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 4; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - - insn[len - 4] = __BPF_STMT(BPF_MISC | BPF_TAX, 0); - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns8(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i, jmp_off = len - 3; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_JUMP(BPF_JMP | BPF_JGT, 0xffffffff, jmp_off--, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns9(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_JMP_IMM(BPF_JA, 0, 0, len - 2); - insn[1] = BPF_ALU32_IMM(BPF_MOV, R0, 0xcbababab); - insn[2] = BPF_EXIT_INSN(); - - for (i = 3; i < len - 2; i++) - insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xfefefefe); - - insn[len - 2] = BPF_EXIT_INSN(); - insn[len - 1] = BPF_JMP_IMM(BPF_JA, 0, 0, -(len - 1)); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns10(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS, hlen = len - 2; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < hlen / 2; i++) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 2 - 2 * i); - for (i = hlen - 1; i > hlen / 2; i--) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 1 - 2 * i); - - insn[hlen / 2] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen / 2 - 1); - insn[hlen] = BPF_ALU32_IMM(BPF_MOV, R0, 0xabababac); - insn[hlen + 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, - unsigned int plen) -{ - struct sock_filter *insn; - unsigned int rlen; - int i, j; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - rlen = (len % plen) - 1; - - for (i = 0; i + plen < len; i += plen) - for (j = 0; j < plen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, - plen - 1 - j, 0, 0); - for (j = 0; j < rlen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, rlen - 1 - j, - 0, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xababcbac); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns11(struct bpf_test *self) -{ - /* Hits 70 passes on x86_64 and triggers NOPs padding. */ - return __bpf_fill_ja(self, BPF_MAXINSNS, 68); -} - -static int bpf_fill_maxinsns12(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns13(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 3; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_IMM, 0xabababab); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_ja(struct bpf_test *self) -{ - /* Hits exactly 11 passes on x86_64 JIT. */ - return __bpf_fill_ja(self, 12, 9); -} - -static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i += 2) { - insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0); - insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_stxdw(struct bpf_test *self, int size) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_ALU32_IMM(BPF_MOV, R0, 1); - insn[1] = BPF_ST_MEM(size, R10, -40, 42); - - for (i = 2; i < len - 2; i++) - insn[i] = BPF_STX_XADD(size, R10, R0, -40); - - insn[len - 2] = BPF_LDX_MEM(size, R0, R10, -40); - insn[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - self->stack_depth = 40; - - return 0; -} - -static int bpf_fill_stxw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_W); -} - -static int bpf_fill_stxdw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_DW); -} - -static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) -{ - struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; - - memcpy(insns, tmp, sizeof(tmp)); - return 2; -} - -/* - * Branch conversion tests. Complex operations can expand to a lot - * of instructions when JITed. This in turn may cause jump offsets - * to overflow the field size of the native instruction, triggering - * a branch conversion mechanism in some JITs. - */ -static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm, bool alu32) -{ - struct bpf_insn *insns; - int len = S16_MAX + 5; - int i; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); - insns[i++] = BPF_EXIT_INSN(); - - while (i < len - 1) { - static const int ops[] = { - BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, - BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, - }; - int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; - - if ((i & 1) || alu32) - insns[i++] = BPF_ALU32_REG(op, R0, R1); - else - insns[i++] = BPF_ALU64_REG(op, R0, R1); - } - - insns[i++] = BPF_EXIT_INSN(); - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* Branch taken by runtime decision */ -static int bpf_fill_max_jmp_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, true); -} - -static int bpf_fill_max_jmp_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, false); -} - -/* Branch not taken by runtime decision */ -static int bpf_fill_max_jmp_not_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, true); -} - -static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, false); -} - -/* Branch always taken, known at JIT time */ -static int bpf_fill_max_jmp_always_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, true); -} - -static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, false); -} - -/* Branch never taken, known at JIT time */ -static int bpf_fill_max_jmp_never_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, true); -} - -static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, false); -} - -/* ALU result computation used in tests */ -enum { F_ALU32 = 1, F_SIGNED = 2 }; - -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags) -{ - bool is_signed = flags & F_SIGNED; - - /* Narrow operands for ALU32 */ - if (flags & F_ALU32) { - if (is_signed) { - v1 = (u64)(s32)v1; - v2 = (u64)(s32)v2; - } else { - v1 = (u32)v1; - v2 = (u32)v2; - } - } - - *res = 0; - switch (op) { - case BPF_MOV: - *res = v2; - break; - case BPF_AND: - *res = v1 & v2; - break; - case BPF_OR: - *res = v1 | v2; - break; - case BPF_XOR: - *res = v1 ^ v2; - break; - case BPF_LSH: - *res = v1 << v2; - break; - case BPF_RSH: - *res = v1 >> v2; - break; - case BPF_ARSH: - *res = v1 >> v2; - if (v2 > 0 && v1 > S64_MAX) - *res |= ~0ULL << (64 - v2); - break; - case BPF_ADD: - *res = v1 + v2; - break; - case BPF_SUB: - *res = v1 - v2; - break; - case BPF_MUL: - *res = v1 * v2; - break; - case BPF_DIV: - if (v2 == 0) - return false; - if (!is_signed) { - *res = div64_u64(v1, v2); - } else { - if ((s64)v2 == -1) /* Handled by verifier */ - return false; - *res = (u64)div64_s64(v1, v2); - } - break; - case BPF_MOD: - if (v2 == 0) - return false; - if (!is_signed) { - div64_u64_rem(v1, v2, res); - } else { - if ((s64)v2 == -1) - return false; - /* - * Avoid s64 % s64 which generates __moddi3 on - * 32-bit architectures. Use div64_s64 instead. - */ - *res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2); - } - break; - } - return true; -} - -/* Test an ALU shift operation for all valid shift values */ -static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, - u8 mode, u32 flags) -{ - static const s64 regs[] = { - 0x0123456789abcdefLL, /* dword > 0, word < 0 */ - 0xfedcba9876543210LL, /* dword < 0, word > 0 */ - 0xfedcba0198765432LL, /* dword < 0, word < 0 */ - 0x0123458967abcdefLL, /* dword > 0, word > 0 */ - }; - int bits = (flags & F_ALU32) ? 32 : 64; - int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; - struct bpf_insn *insn; - int imm, k; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (k = 0; k < ARRAY_SIZE(regs); k++) { - s64 reg = regs[k]; - - i += __bpf_ld_imm64(&insn[i], R3, reg); - - for (imm = 0; imm < bits; imm++) { - u64 val; - - /* Perform operation */ - insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); - if (flags & F_ALU32) { - if (mode == BPF_K) - insn[i++] = BPF_ALU32_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU32_REG(op, R1, R2); - - if (op == BPF_ARSH) - reg = (s32)reg; - else - reg = (u32)reg; - __bpf_alu_result(&val, reg, imm, op, 0); - val = (u32)val; - } else { - if (mode == BPF_K) - insn[i++] = BPF_ALU64_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU64_REG(op, R1, R2); - __bpf_alu_result(&val, reg, imm, op, 0); - } - - /* - * When debugging a JIT that fails this test, one - * can write the immediate value to R0 here to find - * out which operand values that fail. - */ - - /* Load reference and check the result */ - i += __bpf_ld_imm64(&insn[i], R4, val); - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0); -} - -static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0); -} - -static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0); -} - -static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0); -} - -static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0); -} - -static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0); -} - -static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32); -} - -/* - * Test an ALU register shift operation for all valid shift values - * for the case when the source and destination are the same. - */ -static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, - u32 flags) -{ - int bits = (flags & F_ALU32) ? 32 : 64; - int len = 3 + 6 * bits; - struct bpf_insn *insn; - int i = 0; - u64 val; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (val = 0; val < bits; val++) { - u64 res; - - /* Perform operation */ - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); - if (flags & F_ALU32) - insn[i++] = BPF_ALU32_REG(op, R1, R1); - else - insn[i++] = BPF_ALU64_REG(op, R1, R1); - - /* Compute the reference result */ - __bpf_alu_result(&res, val, val, op, 0); - if (flags & F_ALU32) - res = (u32)res; - i += __bpf_ld_imm64(&insn[i], R2, res); - - /* Check the actual result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32); -} - -/* - * Common operand pattern generator for exhaustive power-of-two magnitudes - * tests. The block size parameters can be adjusted to increase/reduce the - * number of combinatons tested and thereby execution speed and memory - * footprint. - */ - -static inline s64 value(int msb, int delta, int sign) -{ - return sign * (1LL << msb) + delta; -} - -static int __bpf_fill_pattern(struct bpf_test *self, void *arg, - int dbits, int sbits, int block1, int block2, - int (*emit)(struct bpf_test*, void*, - struct bpf_insn*, s64, s64)) -{ - static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; - struct bpf_insn *insns; - int di, si, bt, db, sb; - int count, len, k; - int extra = 1 + 2; - int i = 0; - - /* Total number of iterations for the two pattern */ - count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); - count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); - - /* Compute the maximum number of insns and allocate the buffer */ - len = extra + count * (*emit)(self, arg, NULL, 0, 0); - insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); - if (!insns) - return -ENOMEM; - - /* Add head instruction(s) */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - /* - * Pattern 1: all combinations of power-of-two magnitudes and sign, - * and with a block of contiguous values around each magnitude. - */ - for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ - for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block1 / 2); - db < (block1 + 1) / 2; db++) - for (sb = -(block1 / 2); - sb < (block1 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(di, db, sgn[k][0]); - src = value(si, sb, sgn[k][1]); - i += (*emit)(self, arg, - &insns[i], - dst, src); - } - /* - * Pattern 2: all combinations for a larger block of values - * for each power-of-two magnitude and sign, where the magnitude is - * the same for both operands. - */ - for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) - for (sb = -(block2 / 2); - sb < (block2 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(bt % dbits, db, sgn[k][0]); - src = value(bt % sbits, sb, sgn[k][1]); - i += (*emit)(self, arg, &insns[i], - dst, src); - } - - /* Append tail instructions */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_EXIT_INSN(); - BUG_ON(i > len); - - self->u.ptr.insns = insns; - self->u.ptr.len = i; - - return 0; -} - -/* - * Block size parameters used in pattern tests below. une as needed to - * increase/reduce the number combinations tested, see following examples. - * block values per operand MSB - * ---------------------------------------- - * 0 none - * 1 (1 << MSB) - * 2 (1 << MSB) + [-1, 0] - * 3 (1 << MSB) + [-1, 0, 1] - */ -#define PATTERN_BLOCK1 1 -#define PATTERN_BLOCK2 5 - -/* Number of test runs for a pattern test */ -#define NR_PATTERN_RUNS 1 - -/* - * Exhaustive tests of ALU operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the ALU and ALU64 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ -static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, dst, src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_imm); -} - -static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_imm); -} - -static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_reg); -} - -static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_reg); -} - -/* ALU64 immediate operations */ -static int bpf_fill_alu64_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, 0); -} - -/* Signed ALU64 immediate operations */ -static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED); -} - -/* Signed ALU32 immediate operations */ -static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 immediate operations */ -static int bpf_fill_alu32_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu32_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu32_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu32_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu32_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu32_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu32_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu32_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu32_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, 0); -} - -/* ALU64 register operations */ -static int bpf_fill_alu64_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOV); -} - -static int bpf_fill_alu64_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_AND); -} - -static int bpf_fill_alu64_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_OR); -} - -static int bpf_fill_alu64_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_XOR); -} - -static int bpf_fill_alu64_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_ADD); -} - -static int bpf_fill_alu64_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_SUB); -} - -static int bpf_fill_alu64_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MUL); -} - -static int bpf_fill_alu64_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_DIV); -} - -static int bpf_fill_alu64_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOD); -} - -/* ALU32 register operations */ -static int bpf_fill_alu32_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOV); -} - -static int bpf_fill_alu32_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_AND); -} - -static int bpf_fill_alu32_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_OR); -} - -static int bpf_fill_alu32_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_XOR); -} - -static int bpf_fill_alu32_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_ADD); -} - -static int bpf_fill_alu32_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_SUB); -} - -static int bpf_fill_alu32_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MUL); -} - -static int bpf_fill_alu32_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_DIV); -} - -static int bpf_fill_alu32_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOD); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, - u32 flags) -{ - int len = 2 + 10 * 10; - struct bpf_insn *insns; - u64 dst, res; - int i = 0; - u32 imm; - int rd; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (flags & F_SIGNED) { - if (flags & F_ALU32) - dst = -76543210; - else - dst = -7654321076543210LL; - } else { - if (flags & F_ALU32) - dst = 0x76543210U; - else - dst = 0x7edcba9876543210ULL; - } - imm = 0x01234567U; - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - imm &= 31; - - __bpf_alu_result(&res, dst, imm, op, flags); - - if (flags & F_ALU32) - res = (u32)res; - - /* Check all operand registers */ - for (rd = R0; rd <= R9; rd++) { - i += __bpf_ld_imm64(&insns[i], rd, dst); - - s16 off = (flags & F_SIGNED) ? 1 : 0; - - if (flags & F_ALU32) - insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off); - else - insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 K registers */ -static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0); -} - -/* Signed ALU64 K registers */ -static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 K registers */ -static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32); -} - -static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32); -} - -static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32); -} - -static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32); -} - -static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32); -} - -static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32); -} - -static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32); -} - -static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32); -} - -static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32); -} - -static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32); -} - -/* Signed ALU32 K registers */ -static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) -{ - int len = 2 + 10 * 10 * 12; - u64 dst, src, res, same; - struct bpf_insn *insns; - int rd, rs; - int i = 0; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (alu32) { - dst = 0x76543210U; - src = 0x01234567U; - } else { - dst = 0x7edcba9876543210ULL; - src = 0x0123456789abcdefULL; - } - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - src &= 31; - - __bpf_alu_result(&res, dst, src, op, 0); - __bpf_alu_result(&same, src, src, op, 0); - - if (alu32) { - res = (u32)res; - same = (u32)same; - } - - /* Check all combinations of operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 val = rd == rs ? same : res; - - i += __bpf_ld_imm64(&insns[i], rd, dst); - i += __bpf_ld_imm64(&insns[i], rs, src); - - if (alu32) - insns[i++] = BPF_ALU32_REG(op, rd, rs); - else - insns[i++] = BPF_ALU64_REG(op, rd, rs); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 X register combinations */ -static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); -} - -static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); -} - -static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); -} - -static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); -} - -static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); -} - -static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); -} - -static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); -} - -static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); -} - -static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); -} - -static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); -} - -static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); -} - -static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); -} - -/* ALU32 X register combinations */ -static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); -} - -static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); -} - -static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); -} - -static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); -} - -static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); -} - -static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); -} - -static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); -} - -static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); -} - -static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); -} - -static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); -} - -static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); -} - -static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); -} - -/* - * Exhaustive tests of atomic operations for all power-of-two operand - * magnitudes, both for positive and negative values. - */ - -static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, dst, src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = (u32)dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); - insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 23; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 27; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - i += __bpf_ld_imm64(&insns[i], R0, dst); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_fill_atomic64(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic64); -} - -static int __bpf_fill_atomic32(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic32); -} - -/* 64-bit atomic operations */ -static int bpf_fill_atomic64_add(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD); -} - -static int bpf_fill_atomic64_and(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND); -} - -static int bpf_fill_atomic64_or(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR); -} - -static int bpf_fill_atomic64_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg64(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg64); -} - -/* 32-bit atomic operations */ -static int bpf_fill_atomic32_add(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD); -} - -static int bpf_fill_atomic32_and(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND); -} - -static int bpf_fill_atomic32_or(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR); -} - -static int bpf_fill_atomic32_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg32(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg32); -} - -/* - * Test JITs that implement ATOMIC operations as function calls or - * other primitives, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) -{ - struct bpf_insn *insn; - int len = 2 + 34 * 10 * 10; - u64 mem, upd, res; - int rd, rs, i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - /* Operand and memory values */ - if (width == BPF_DW) { - mem = 0x0123456789abcdefULL; - upd = 0xfedcba9876543210ULL; - } else { /* BPF_W */ - mem = 0x01234567U; - upd = 0x76543210U; - } - - /* Memory updated according to operation */ - switch (op) { - case BPF_XCHG: - res = upd; - break; - case BPF_CMPXCHG: - res = mem; - break; - default: - __bpf_alu_result(&res, mem, upd, BPF_OP(op), 0); - } - - /* Test all operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 cmp, src; - - /* Initialize value in memory */ - i += __bpf_ld_imm64(&insn[i], R0, mem); - insn[i++] = BPF_STX_MEM(width, R10, R0, -8); - - /* Initialize registers in order */ - i += __bpf_ld_imm64(&insn[i], R0, ~mem); - i += __bpf_ld_imm64(&insn[i], rs, upd); - insn[i++] = BPF_MOV64_REG(rd, R10); - - /* Perform atomic operation */ - insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); - if (op == BPF_CMPXCHG && width == BPF_W) - insn[i++] = BPF_ZEXT_REG(R0); - - /* Check R0 register value */ - if (op == BPF_CMPXCHG) - cmp = mem; /* Expect value from memory */ - else if (R0 == rd || R0 == rs) - cmp = 0; /* Aliased, checked below */ - else - cmp = ~mem; /* Expect value to be preserved */ - if (cmp) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - (u32)cmp, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - cmp >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check source register value */ - if (rs == R0 && op == BPF_CMPXCHG) - src = 0; /* Aliased with R0, checked above */ - else if (rs == rd && (op == BPF_CMPXCHG || - !(op & BPF_FETCH))) - src = 0; /* Aliased with rd, checked below */ - else if (op == BPF_CMPXCHG) - src = upd; /* Expect value to be preserved */ - else if (op & BPF_FETCH) - src = mem; /* Expect fetched value from mem */ - else /* no fetch */ - src = upd; /* Expect value to be preserved */ - if (src) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - (u32)src, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - src >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check destination register value */ - if (!(rd == R0 && op == BPF_CMPXCHG) && - !(rd == rs && (op & BPF_FETCH))) { - insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check value in memory */ - if (rs != rd) { /* No aliasing */ - i += __bpf_ld_imm64(&insn[i], R1, res); - } else if (op == BPF_XCHG) { /* Aliased, XCHG */ - insn[i++] = BPF_MOV64_REG(R1, R10); - } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - } else { /* Aliased, ALU oper */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); - } - - insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); - if (width == BPF_DW) - insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - else /* width == BPF_W */ - insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_MOV64_IMM(R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = i; - BUG_ON(i > len); - - return 0; -} - -/* 64-bit atomic register tests */ -static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); -} - -static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); -} - -static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); -} - -static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); -} - -static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); -} - -/* 32-bit atomic register tests */ -static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); -} - -static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); -} - -static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); -} - -static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); -} - -static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); -} - -/* - * Test the two-instruction 64-bit immediate load operation for all - * power-of-two magnitudes of the immediate operand. For each MSB, a block - * of immediate values centered around the power-of-two MSB are tested, - * both for positive and negative values. The test is designed to verify - * the operation for JITs that emit different code depending on the magnitude - * of the immediate value. This is often the case if the native instruction - * immediate field width is narrower than 32 bits. - */ -static int bpf_fill_ld_imm64_magn(struct bpf_test *self) -{ - int block = 64; /* Increase for more tests per MSB position */ - int len = 3 + 8 * 63 * block * 2; - struct bpf_insn *insn; - int bit, adj, sign; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (bit = 0; bit <= 62; bit++) { - for (adj = -block / 2; adj < block / 2; adj++) { - for (sign = -1; sign <= 1; sign += 2) { - s64 imm = sign * ((1LL << bit) + adj); - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, - (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* - * Test the two-instruction 64-bit immediate load operation for different - * combinations of bytes. Each byte in the 64-bit word is constructed as - * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. - * All patterns (base1, mask1) and (base2, mask2) bytes are tested. - */ -static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, - u8 base1, u8 mask1, - u8 base2, u8 mask2) -{ - struct bpf_insn *insn; - int len = 3 + 8 * BIT(8); - int pattern, index; - u32 rand = 1; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (pattern = 0; pattern < BIT(8); pattern++) { - u64 imm = 0; - - for (index = 0; index < 8; index++) { - int byte; - - if (pattern & BIT(index)) - byte = (base1 & mask1) | (rand & ~mask1); - else - byte = (base2 & mask2) | (rand & ~mask2); - imm = (imm << 8) | byte; - } - - /* Update our LCG */ - rand = rand * 1664525 + 1013904223; - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_ld_imm64_checker(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); -} - -static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); -} - -static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); -} - -static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); -} - -/* - * Exhaustive tests of JMP operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the JMP and JMP32 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ - -static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) -{ - switch (op) { - case BPF_JSET: - return !!(v1 & v2); - case BPF_JEQ: - return v1 == v2; - case BPF_JNE: - return v1 != v2; - case BPF_JGT: - return (u64)v1 > (u64)v2; - case BPF_JGE: - return (u64)v1 >= (u64)v2; - case BPF_JLT: - return (u64)v1 < (u64)v2; - case BPF_JLE: - return (u64)v1 <= (u64)v2; - case BPF_JSGT: - return v1 > v2; - case BPF_JSGE: - return v1 >= v2; - case BPF_JSLT: - return v1 < v2; - case BPF_JSLE: - return v1 <= v2; - } - return false; -} - -static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); - int i = 0; - - insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5 + 1; -} - -static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5; -} - -static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_imm); -} - -static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_imm); -} - -static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_reg); -} - -static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_reg); -} - -/* JMP immediate tests */ -static int bpf_fill_jmp_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLE); -} - -/* JMP32 immediate tests */ -static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLE); -} - -/* JMP register tests */ -static int bpf_fill_jmp_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLE); -} - -/* JMP32 register tests */ -static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLE); -} - -/* - * Set up a sequence of staggered jumps, forwards and backwards with - * increasing offset. This tests the conversion of relative jumps to - * JITed native jumps. On some architectures, for example MIPS, a large - * PC-relative jump offset may overflow the immediate field of the native - * conditional branch instruction, triggering a conversion to use an - * absolute jump instead. Since this changes the jump offsets, another - * offset computation pass is necessary, and that may in turn trigger - * another branch conversion. This jump sequence is particularly nasty - * in that regard. - * - * The sequence generation is parameterized by size and jump type. - * The size must be even, and the expected result is always size + 1. - * Below is an example with size=8 and result=9. - * - * ________________________Start - * R0 = 0 - * R1 = r1 - * R2 = r2 - * ,------- JMP +4 * 3______________Preamble: 4 insns - * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. - * | | R0 = 8 | - * | | JMP +7 * 3 ------------------------. - * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | - * | | | R0 = 6 | | | - * | | | JMP +5 * 3 ------------------. | | - * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | - * | | | | R0 = 4 | | | | | - * | | | | JMP +3 * 3 ------------. | | | | - * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | - * | | | | | R0 = 2 | | | | | | | - * | | | | | JMP +1 * 3 ------. | | | | | | - * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc - * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off - * | | | | | JMP -2 * 3 ---' | | | | | | | - * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | - * | | | | | | R0 = 3 | | | | | | - * | | | | | | JMP -4 * 3 ---------' | | | | | - * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | - * | | | | | | | R0 = 5 | | | | - * | | | | | | | JMP -6 * 3 ---------------' | | | - * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | - * | | | | | | | | R0 = 7 | | - * | | Error | | | JMP -8 * 3 ---------------------' | - * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' - * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns - * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn - * - */ - -/* The maximum size parameter */ -#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) - -/* We use a reduced number of iterations to get a reasonable execution time */ -#define NR_STAGGERED_JMP_RUNS 10 - -static int __bpf_fill_staggered_jumps(struct bpf_test *self, - const struct bpf_insn *jmp, - u64 r1, u64 r2) -{ - int size = self->test[0].result - 1; - int len = 4 + 3 * (size + 1); - struct bpf_insn *insns; - int off, ind; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Preamble */ - insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); - insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); - insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); - - /* Sequence */ - for (ind = 0, off = size; ind <= size; ind++, off -= 2) { - struct bpf_insn *ins = &insns[4 + 3 * ind]; - int loc; - - if (off == 0) - off--; - - loc = abs(off); - ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, - 3 * (size - ind) + 1); - ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); - ins[2] = *jmp; - ins[2].off = 3 * (off - 1); - } - - /* Return */ - insns[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - - return 0; -} - -/* 64-bit unconditional jump */ -static int bpf_fill_staggered_ja(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); -} - -/* 64-bit immediate jumps */ -static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 64-bit register jumps */ -static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - -/* 32-bit immediate jumps */ -static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 32-bit register jumps */ -static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - - -static struct bpf_test tests[] = { - { - "TAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_NEG, 0), /* A == -3 */ - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), /* X == len - 3 */ - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 2, 10 }, { 3, 20 }, { 4, 30 } }, - }, - { - "TXA", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) /* A == len * 2 */ - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 1, 2 }, { 3, 6 }, { 4, 8 } }, - }, - { - "ADD_SUB_MUL_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 3), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfffffffd } } - }, - { - "DIV_MOD_KX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 8), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x70000000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x70000000), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x20000000 } } - }, - { - "AND_OR_LSH_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xff), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 27), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xf), - BPF_STMT(BPF_ALU | BPF_OR | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x800000ff }, { 1, 0x800000ff } }, - }, - { - "LD_IMM_0", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0), /* ld #0 */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 1 } }, - }, - { - "LD_IND", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, MAX_K), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, 1000), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3 }, - { { 1, 0 }, { 2, 3 } }, - }, - { - "LD_IND_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_LL_OFF - 1), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3, 0xff }, - { { 1, 1 }, { 3, 3 }, { 4, 0xff } }, - }, - { - "LD_ABS_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 15, 0 }, { 16, 3 } }, - }, - { - "LD_IND_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_NET_OFF - 15), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 14, 0 }, { 15, 1 }, { 17, 3 } }, - }, - { - "LD_PKTTYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 3 }, { 10, 3 } }, - }, - { - "LD_MARK", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MARK), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_MARK}, { 10, SKB_MARK} }, - }, - { - "LD_RXHASH", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_RXHASH), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_HASH}, { 10, SKB_HASH} }, - }, - { - "LD_QUEUE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_QUEUE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_QUEUE_MAP }, { 10, SKB_QUEUE_MAP } }, - }, - { - "LD_PROTOCOL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 1), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 20, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PROTOCOL), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 30, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30 }, - { { 10, ETH_P_IP }, { 100, ETH_P_IP } }, - }, - { - "LD_VLAN_TAG", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_TCI }, - { 10, SKB_VLAN_TCI } - }, - }, - { - "LD_VLAN_TAG_PRESENT", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - }, - { - "LD_IFINDEX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_IFINDEX), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_IFINDEX }, { 10, SKB_DEV_IFINDEX } }, - }, - { - "LD_HATYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_HATYPE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_TYPE }, { 10, SKB_DEV_TYPE } }, - }, - { - "LD_CPU", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 } }, - }, - { - "LD_NLATTR", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 6 } }, - }, - { - "LD_NLATTR_NEST", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 10 } }, - }, - { - "LD_PAYLOAD_OFF", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - /* 00:00:00:00:00:00 > 00:00:00:00:00:00, ethtype IPv4 (0x0800), - * length 98: 127.0.0.1 > 127.0.0.1: ICMP echo request, - * id 9737, seq 1, length 64 - */ - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x54, 0xac, 0x8b, 0x40, 0x00, 0x40, - 0x01, 0x90, 0x1b, 0x7f, 0x00, 0x00, 0x01 }, - { { 30, 0 }, { 100, 42 } }, - }, - { - "LD_ANC_XOR", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 300), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_ALU_XOR_X), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 4, 0xA ^ 300 }, { 20, 0xA ^ 300 } }, - }, - { - "SPILL_FILL", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_RSH, 1), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_ST, 1), /* M1 = 1 ^ len */ - BPF_STMT(BPF_ALU | BPF_XOR | BPF_K, 0x80000000), - BPF_STMT(BPF_ST, 2), /* M2 = 1 ^ len ^ 0x80000000 */ - BPF_STMT(BPF_STX, 15), /* M3 = len */ - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_LD | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0x80000001 }, { 2, 0x80000002 }, { 60, 0x80000000 ^ 60 } } - }, - { - "JEQ", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 3, 3, 3, 3, 3 }, - { { 1, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGT", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGE (jt 0), test 1", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, 1 } }, - }, - { - "JGE (jt 0), test 2", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 5, 3, 3 }, - { { 4, 1 }, { 5, 1 }, { 6, MAX_K } }, - }, - { - "JGE", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, MAX_K), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 1, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 2, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 3, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 4, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 40), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 1, 2, 3, 4, 5 }, - { { 1, 20 }, { 3, 40 }, { 5, MAX_K } }, - }, - { - "JSET", - .u.insns = { - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 1, 1, 1), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, 4), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_IND, 0), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 1, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x80000000, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 0, 0xAA, 0x55, 1 }, - { { 4, 10 }, { 5, 20 }, { 6, MAX_K } }, - }, - { - "tcpdump port 22", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 0, 8), /* IPv6 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 17), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 54), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 14, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 56), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 12, 13), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0800, 0, 12), /* IPv4 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 8), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 6, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - /* 3c:07:54:43:e5:76 > 10:bf:48:d6:43:d6, ethertype IPv4(0x0800) - * length 114: 10.1.1.149.49700 > 10.1.2.10.22: Flags [P.], - * seq 1305692979:1305693027, ack 3650467037, win 65535, - * options [nop,nop,TS val 2502645400 ecr 3971138], length 48 - */ - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "tcpdump complex", - .u.insns = { - /* tcpdump -nei eth0 'tcp port 22 and (((ip[2:2] - - * ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0) and - * (len > 115 or len < 30000000000)' -d - */ - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 30, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x800, 0, 29), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 0, 27), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 25, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 20), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 16), - BPF_STMT(BPF_ST, 1), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 14), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x5), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ST, 5), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 26), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_RSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x9), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 5), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 4, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, 0x73, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0xfc23ac00, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "RET_A", - .u.insns = { - /* check that uninitialized X and A contain zeros */ - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { {1, 0}, {2, 0} }, - }, - { - "INT: ADD trivial", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_IMM(BPF_ADD, R1, -1), - BPF_ALU64_IMM(BPF_MUL, R1, 3), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffd } } - }, - { - "INT: MUL_X", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_JMP_IMM(BPF_JEQ, R1, 0xfffffffd, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL_X2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0x2ffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL32_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU32_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0xffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - /* Have to test all register combinations, since - * JITing of different registers will produce - * different asm code. - */ - "INT: ADD 64-bit", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R0, 20), - BPF_ALU64_IMM(BPF_ADD, R1, 20), - BPF_ALU64_IMM(BPF_ADD, R2, 20), - BPF_ALU64_IMM(BPF_ADD, R3, 20), - BPF_ALU64_IMM(BPF_ADD, R4, 20), - BPF_ALU64_IMM(BPF_ADD, R5, 20), - BPF_ALU64_IMM(BPF_ADD, R6, 20), - BPF_ALU64_IMM(BPF_ADD, R7, 20), - BPF_ALU64_IMM(BPF_ADD, R8, 20), - BPF_ALU64_IMM(BPF_ADD, R9, 20), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R1, R0), - BPF_ALU64_REG(BPF_ADD, R1, R1), - BPF_ALU64_REG(BPF_ADD, R1, R2), - BPF_ALU64_REG(BPF_ADD, R1, R3), - BPF_ALU64_REG(BPF_ADD, R1, R4), - BPF_ALU64_REG(BPF_ADD, R1, R5), - BPF_ALU64_REG(BPF_ADD, R1, R6), - BPF_ALU64_REG(BPF_ADD, R1, R7), - BPF_ALU64_REG(BPF_ADD, R1, R8), - BPF_ALU64_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R2, R0), - BPF_ALU64_REG(BPF_ADD, R2, R1), - BPF_ALU64_REG(BPF_ADD, R2, R2), - BPF_ALU64_REG(BPF_ADD, R2, R3), - BPF_ALU64_REG(BPF_ADD, R2, R4), - BPF_ALU64_REG(BPF_ADD, R2, R5), - BPF_ALU64_REG(BPF_ADD, R2, R6), - BPF_ALU64_REG(BPF_ADD, R2, R7), - BPF_ALU64_REG(BPF_ADD, R2, R8), - BPF_ALU64_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R3, R0), - BPF_ALU64_REG(BPF_ADD, R3, R1), - BPF_ALU64_REG(BPF_ADD, R3, R2), - BPF_ALU64_REG(BPF_ADD, R3, R3), - BPF_ALU64_REG(BPF_ADD, R3, R4), - BPF_ALU64_REG(BPF_ADD, R3, R5), - BPF_ALU64_REG(BPF_ADD, R3, R6), - BPF_ALU64_REG(BPF_ADD, R3, R7), - BPF_ALU64_REG(BPF_ADD, R3, R8), - BPF_ALU64_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R4, R0), - BPF_ALU64_REG(BPF_ADD, R4, R1), - BPF_ALU64_REG(BPF_ADD, R4, R2), - BPF_ALU64_REG(BPF_ADD, R4, R3), - BPF_ALU64_REG(BPF_ADD, R4, R4), - BPF_ALU64_REG(BPF_ADD, R4, R5), - BPF_ALU64_REG(BPF_ADD, R4, R6), - BPF_ALU64_REG(BPF_ADD, R4, R7), - BPF_ALU64_REG(BPF_ADD, R4, R8), - BPF_ALU64_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R5, R0), - BPF_ALU64_REG(BPF_ADD, R5, R1), - BPF_ALU64_REG(BPF_ADD, R5, R2), - BPF_ALU64_REG(BPF_ADD, R5, R3), - BPF_ALU64_REG(BPF_ADD, R5, R4), - BPF_ALU64_REG(BPF_ADD, R5, R5), - BPF_ALU64_REG(BPF_ADD, R5, R6), - BPF_ALU64_REG(BPF_ADD, R5, R7), - BPF_ALU64_REG(BPF_ADD, R5, R8), - BPF_ALU64_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R6, R0), - BPF_ALU64_REG(BPF_ADD, R6, R1), - BPF_ALU64_REG(BPF_ADD, R6, R2), - BPF_ALU64_REG(BPF_ADD, R6, R3), - BPF_ALU64_REG(BPF_ADD, R6, R4), - BPF_ALU64_REG(BPF_ADD, R6, R5), - BPF_ALU64_REG(BPF_ADD, R6, R6), - BPF_ALU64_REG(BPF_ADD, R6, R7), - BPF_ALU64_REG(BPF_ADD, R6, R8), - BPF_ALU64_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R7, R0), - BPF_ALU64_REG(BPF_ADD, R7, R1), - BPF_ALU64_REG(BPF_ADD, R7, R2), - BPF_ALU64_REG(BPF_ADD, R7, R3), - BPF_ALU64_REG(BPF_ADD, R7, R4), - BPF_ALU64_REG(BPF_ADD, R7, R5), - BPF_ALU64_REG(BPF_ADD, R7, R6), - BPF_ALU64_REG(BPF_ADD, R7, R7), - BPF_ALU64_REG(BPF_ADD, R7, R8), - BPF_ALU64_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R8, R0), - BPF_ALU64_REG(BPF_ADD, R8, R1), - BPF_ALU64_REG(BPF_ADD, R8, R2), - BPF_ALU64_REG(BPF_ADD, R8, R3), - BPF_ALU64_REG(BPF_ADD, R8, R4), - BPF_ALU64_REG(BPF_ADD, R8, R5), - BPF_ALU64_REG(BPF_ADD, R8, R6), - BPF_ALU64_REG(BPF_ADD, R8, R7), - BPF_ALU64_REG(BPF_ADD, R8, R8), - BPF_ALU64_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R9, R0), - BPF_ALU64_REG(BPF_ADD, R9, R1), - BPF_ALU64_REG(BPF_ADD, R9, R2), - BPF_ALU64_REG(BPF_ADD, R9, R3), - BPF_ALU64_REG(BPF_ADD, R9, R4), - BPF_ALU64_REG(BPF_ADD, R9, R5), - BPF_ALU64_REG(BPF_ADD, R9, R6), - BPF_ALU64_REG(BPF_ADD, R9, R7), - BPF_ALU64_REG(BPF_ADD, R9, R8), - BPF_ALU64_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU64_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { - "INT: ADD 32-bit", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 20), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R2, 2), - BPF_ALU32_IMM(BPF_MOV, R3, 3), - BPF_ALU32_IMM(BPF_MOV, R4, 4), - BPF_ALU32_IMM(BPF_MOV, R5, 5), - BPF_ALU32_IMM(BPF_MOV, R6, 6), - BPF_ALU32_IMM(BPF_MOV, R7, 7), - BPF_ALU32_IMM(BPF_MOV, R8, 8), - BPF_ALU32_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R1, 10), - BPF_ALU64_IMM(BPF_ADD, R2, 10), - BPF_ALU64_IMM(BPF_ADD, R3, 10), - BPF_ALU64_IMM(BPF_ADD, R4, 10), - BPF_ALU64_IMM(BPF_ADD, R5, 10), - BPF_ALU64_IMM(BPF_ADD, R6, 10), - BPF_ALU64_IMM(BPF_ADD, R7, 10), - BPF_ALU64_IMM(BPF_ADD, R8, 10), - BPF_ALU64_IMM(BPF_ADD, R9, 10), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_ALU32_REG(BPF_ADD, R0, R2), - BPF_ALU32_REG(BPF_ADD, R0, R3), - BPF_ALU32_REG(BPF_ADD, R0, R4), - BPF_ALU32_REG(BPF_ADD, R0, R5), - BPF_ALU32_REG(BPF_ADD, R0, R6), - BPF_ALU32_REG(BPF_ADD, R0, R7), - BPF_ALU32_REG(BPF_ADD, R0, R8), - BPF_ALU32_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R1, R0), - BPF_ALU32_REG(BPF_ADD, R1, R1), - BPF_ALU32_REG(BPF_ADD, R1, R2), - BPF_ALU32_REG(BPF_ADD, R1, R3), - BPF_ALU32_REG(BPF_ADD, R1, R4), - BPF_ALU32_REG(BPF_ADD, R1, R5), - BPF_ALU32_REG(BPF_ADD, R1, R6), - BPF_ALU32_REG(BPF_ADD, R1, R7), - BPF_ALU32_REG(BPF_ADD, R1, R8), - BPF_ALU32_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R2, R0), - BPF_ALU32_REG(BPF_ADD, R2, R1), - BPF_ALU32_REG(BPF_ADD, R2, R2), - BPF_ALU32_REG(BPF_ADD, R2, R3), - BPF_ALU32_REG(BPF_ADD, R2, R4), - BPF_ALU32_REG(BPF_ADD, R2, R5), - BPF_ALU32_REG(BPF_ADD, R2, R6), - BPF_ALU32_REG(BPF_ADD, R2, R7), - BPF_ALU32_REG(BPF_ADD, R2, R8), - BPF_ALU32_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R3, R0), - BPF_ALU32_REG(BPF_ADD, R3, R1), - BPF_ALU32_REG(BPF_ADD, R3, R2), - BPF_ALU32_REG(BPF_ADD, R3, R3), - BPF_ALU32_REG(BPF_ADD, R3, R4), - BPF_ALU32_REG(BPF_ADD, R3, R5), - BPF_ALU32_REG(BPF_ADD, R3, R6), - BPF_ALU32_REG(BPF_ADD, R3, R7), - BPF_ALU32_REG(BPF_ADD, R3, R8), - BPF_ALU32_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R4, R0), - BPF_ALU32_REG(BPF_ADD, R4, R1), - BPF_ALU32_REG(BPF_ADD, R4, R2), - BPF_ALU32_REG(BPF_ADD, R4, R3), - BPF_ALU32_REG(BPF_ADD, R4, R4), - BPF_ALU32_REG(BPF_ADD, R4, R5), - BPF_ALU32_REG(BPF_ADD, R4, R6), - BPF_ALU32_REG(BPF_ADD, R4, R7), - BPF_ALU32_REG(BPF_ADD, R4, R8), - BPF_ALU32_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R5, R0), - BPF_ALU32_REG(BPF_ADD, R5, R1), - BPF_ALU32_REG(BPF_ADD, R5, R2), - BPF_ALU32_REG(BPF_ADD, R5, R3), - BPF_ALU32_REG(BPF_ADD, R5, R4), - BPF_ALU32_REG(BPF_ADD, R5, R5), - BPF_ALU32_REG(BPF_ADD, R5, R6), - BPF_ALU32_REG(BPF_ADD, R5, R7), - BPF_ALU32_REG(BPF_ADD, R5, R8), - BPF_ALU32_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R6, R0), - BPF_ALU32_REG(BPF_ADD, R6, R1), - BPF_ALU32_REG(BPF_ADD, R6, R2), - BPF_ALU32_REG(BPF_ADD, R6, R3), - BPF_ALU32_REG(BPF_ADD, R6, R4), - BPF_ALU32_REG(BPF_ADD, R6, R5), - BPF_ALU32_REG(BPF_ADD, R6, R6), - BPF_ALU32_REG(BPF_ADD, R6, R7), - BPF_ALU32_REG(BPF_ADD, R6, R8), - BPF_ALU32_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R7, R0), - BPF_ALU32_REG(BPF_ADD, R7, R1), - BPF_ALU32_REG(BPF_ADD, R7, R2), - BPF_ALU32_REG(BPF_ADD, R7, R3), - BPF_ALU32_REG(BPF_ADD, R7, R4), - BPF_ALU32_REG(BPF_ADD, R7, R5), - BPF_ALU32_REG(BPF_ADD, R7, R6), - BPF_ALU32_REG(BPF_ADD, R7, R7), - BPF_ALU32_REG(BPF_ADD, R7, R8), - BPF_ALU32_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R8, R0), - BPF_ALU32_REG(BPF_ADD, R8, R1), - BPF_ALU32_REG(BPF_ADD, R8, R2), - BPF_ALU32_REG(BPF_ADD, R8, R3), - BPF_ALU32_REG(BPF_ADD, R8, R4), - BPF_ALU32_REG(BPF_ADD, R8, R5), - BPF_ALU32_REG(BPF_ADD, R8, R6), - BPF_ALU32_REG(BPF_ADD, R8, R7), - BPF_ALU32_REG(BPF_ADD, R8, R8), - BPF_ALU32_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R9, R0), - BPF_ALU32_REG(BPF_ADD, R9, R1), - BPF_ALU32_REG(BPF_ADD, R9, R2), - BPF_ALU32_REG(BPF_ADD, R9, R3), - BPF_ALU32_REG(BPF_ADD, R9, R4), - BPF_ALU32_REG(BPF_ADD, R9, R5), - BPF_ALU32_REG(BPF_ADD, R9, R6), - BPF_ALU32_REG(BPF_ADD, R9, R7), - BPF_ALU32_REG(BPF_ADD, R9, R8), - BPF_ALU32_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU32_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { /* Mainly checking JIT here. */ - "INT: SUB", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, -55, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R0), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_SUB, R1, R3), - BPF_ALU64_REG(BPF_SUB, R1, R4), - BPF_ALU64_REG(BPF_SUB, R1, R5), - BPF_ALU64_REG(BPF_SUB, R1, R6), - BPF_ALU64_REG(BPF_SUB, R1, R7), - BPF_ALU64_REG(BPF_SUB, R1, R8), - BPF_ALU64_REG(BPF_SUB, R1, R9), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_REG(BPF_SUB, R2, R0), - BPF_ALU64_REG(BPF_SUB, R2, R1), - BPF_ALU64_REG(BPF_SUB, R2, R3), - BPF_ALU64_REG(BPF_SUB, R2, R4), - BPF_ALU64_REG(BPF_SUB, R2, R5), - BPF_ALU64_REG(BPF_SUB, R2, R6), - BPF_ALU64_REG(BPF_SUB, R2, R7), - BPF_ALU64_REG(BPF_SUB, R2, R8), - BPF_ALU64_REG(BPF_SUB, R2, R9), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_REG(BPF_SUB, R3, R0), - BPF_ALU64_REG(BPF_SUB, R3, R1), - BPF_ALU64_REG(BPF_SUB, R3, R2), - BPF_ALU64_REG(BPF_SUB, R3, R4), - BPF_ALU64_REG(BPF_SUB, R3, R5), - BPF_ALU64_REG(BPF_SUB, R3, R6), - BPF_ALU64_REG(BPF_SUB, R3, R7), - BPF_ALU64_REG(BPF_SUB, R3, R8), - BPF_ALU64_REG(BPF_SUB, R3, R9), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_REG(BPF_SUB, R4, R0), - BPF_ALU64_REG(BPF_SUB, R4, R1), - BPF_ALU64_REG(BPF_SUB, R4, R2), - BPF_ALU64_REG(BPF_SUB, R4, R3), - BPF_ALU64_REG(BPF_SUB, R4, R5), - BPF_ALU64_REG(BPF_SUB, R4, R6), - BPF_ALU64_REG(BPF_SUB, R4, R7), - BPF_ALU64_REG(BPF_SUB, R4, R8), - BPF_ALU64_REG(BPF_SUB, R4, R9), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_REG(BPF_SUB, R5, R0), - BPF_ALU64_REG(BPF_SUB, R5, R1), - BPF_ALU64_REG(BPF_SUB, R5, R2), - BPF_ALU64_REG(BPF_SUB, R5, R3), - BPF_ALU64_REG(BPF_SUB, R5, R4), - BPF_ALU64_REG(BPF_SUB, R5, R6), - BPF_ALU64_REG(BPF_SUB, R5, R7), - BPF_ALU64_REG(BPF_SUB, R5, R8), - BPF_ALU64_REG(BPF_SUB, R5, R9), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_REG(BPF_SUB, R6, R0), - BPF_ALU64_REG(BPF_SUB, R6, R1), - BPF_ALU64_REG(BPF_SUB, R6, R2), - BPF_ALU64_REG(BPF_SUB, R6, R3), - BPF_ALU64_REG(BPF_SUB, R6, R4), - BPF_ALU64_REG(BPF_SUB, R6, R5), - BPF_ALU64_REG(BPF_SUB, R6, R7), - BPF_ALU64_REG(BPF_SUB, R6, R8), - BPF_ALU64_REG(BPF_SUB, R6, R9), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_REG(BPF_SUB, R7, R0), - BPF_ALU64_REG(BPF_SUB, R7, R1), - BPF_ALU64_REG(BPF_SUB, R7, R2), - BPF_ALU64_REG(BPF_SUB, R7, R3), - BPF_ALU64_REG(BPF_SUB, R7, R4), - BPF_ALU64_REG(BPF_SUB, R7, R5), - BPF_ALU64_REG(BPF_SUB, R7, R6), - BPF_ALU64_REG(BPF_SUB, R7, R8), - BPF_ALU64_REG(BPF_SUB, R7, R9), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_REG(BPF_SUB, R8, R0), - BPF_ALU64_REG(BPF_SUB, R8, R1), - BPF_ALU64_REG(BPF_SUB, R8, R2), - BPF_ALU64_REG(BPF_SUB, R8, R3), - BPF_ALU64_REG(BPF_SUB, R8, R4), - BPF_ALU64_REG(BPF_SUB, R8, R5), - BPF_ALU64_REG(BPF_SUB, R8, R6), - BPF_ALU64_REG(BPF_SUB, R8, R7), - BPF_ALU64_REG(BPF_SUB, R8, R9), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_REG(BPF_SUB, R9, R0), - BPF_ALU64_REG(BPF_SUB, R9, R1), - BPF_ALU64_REG(BPF_SUB, R9, R2), - BPF_ALU64_REG(BPF_SUB, R9, R3), - BPF_ALU64_REG(BPF_SUB, R9, R4), - BPF_ALU64_REG(BPF_SUB, R9, R5), - BPF_ALU64_REG(BPF_SUB, R9, R6), - BPF_ALU64_REG(BPF_SUB, R9, R7), - BPF_ALU64_REG(BPF_SUB, R9, R8), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 11 } } - }, - { /* Mainly checking JIT here. */ - "INT: XOR", - .u.insns_int = { - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_XOR, R1, R1), - BPF_JMP_REG(BPF_JEQ, R0, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R2, R2), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R2, R2), - BPF_ALU64_REG(BPF_XOR, R3, R3), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_JMP_REG(BPF_JEQ, R2, R3, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R3, R3), - BPF_ALU64_REG(BPF_XOR, R4, R4), - BPF_ALU64_IMM(BPF_MOV, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R5, -1), - BPF_JMP_REG(BPF_JEQ, R3, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R4, R4), - BPF_ALU64_REG(BPF_XOR, R5, R5), - BPF_ALU64_IMM(BPF_MOV, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R7, -1), - BPF_JMP_REG(BPF_JEQ, R5, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R5, 1), - BPF_ALU64_REG(BPF_SUB, R5, R5), - BPF_ALU64_REG(BPF_XOR, R6, R6), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R8, -1), - BPF_JMP_REG(BPF_JEQ, R5, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R6, R6), - BPF_ALU64_REG(BPF_XOR, R7, R7), - BPF_JMP_REG(BPF_JEQ, R7, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R7, R7), - BPF_ALU64_REG(BPF_XOR, R8, R8), - BPF_JMP_REG(BPF_JEQ, R7, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R8, R8), - BPF_ALU64_REG(BPF_XOR, R9, R9), - BPF_JMP_REG(BPF_JEQ, R9, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R9, R9), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 2), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { /* Mainly checking JIT here. */ - "INT: MUL", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_MUL, R0, R0), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_REG(BPF_MUL, R0, R2), - BPF_ALU64_REG(BPF_MUL, R0, R3), - BPF_ALU64_REG(BPF_MUL, R0, R4), - BPF_ALU64_REG(BPF_MUL, R0, R5), - BPF_ALU64_REG(BPF_MUL, R0, R6), - BPF_ALU64_REG(BPF_MUL, R0, R7), - BPF_ALU64_REG(BPF_MUL, R0, R8), - BPF_ALU64_REG(BPF_MUL, R0, R9), - BPF_ALU64_IMM(BPF_MUL, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, 439084800, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R1, R0), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_REG(BPF_MUL, R1, R3), - BPF_ALU64_REG(BPF_MUL, R1, R4), - BPF_ALU64_REG(BPF_MUL, R1, R5), - BPF_ALU64_REG(BPF_MUL, R1, R6), - BPF_ALU64_REG(BPF_MUL, R1, R7), - BPF_ALU64_REG(BPF_MUL, R1, R8), - BPF_ALU64_REG(BPF_MUL, R1, R9), - BPF_ALU64_IMM(BPF_MUL, R1, 10), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5a924, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_LSH, R1, 32), - BPF_ALU64_IMM(BPF_ARSH, R1, 32), - BPF_JMP_IMM(BPF_JEQ, R1, 0xebb90000, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R2, R0), - BPF_ALU64_REG(BPF_MUL, R2, R1), - BPF_ALU64_REG(BPF_MUL, R2, R3), - BPF_ALU64_REG(BPF_MUL, R2, R4), - BPF_ALU64_REG(BPF_MUL, R2, R5), - BPF_ALU64_REG(BPF_MUL, R2, R6), - BPF_ALU64_REG(BPF_MUL, R2, R7), - BPF_ALU64_REG(BPF_MUL, R2, R8), - BPF_ALU64_REG(BPF_MUL, R2, R9), - BPF_ALU64_IMM(BPF_MUL, R2, 10), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_MOV, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x35d97ef2 } } - }, - { /* Mainly checking JIT here. */ - "MOV REG64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 0), - BPF_ALU64_IMM(BPF_MOV, R2, 0), - BPF_ALU64_IMM(BPF_MOV, R3, 0), - BPF_ALU64_IMM(BPF_MOV, R4, 0), - BPF_ALU64_IMM(BPF_MOV, R5, 0), - BPF_ALU64_IMM(BPF_MOV, R6, 0), - BPF_ALU64_IMM(BPF_MOV, R7, 0), - BPF_ALU64_IMM(BPF_MOV, R8, 0), - BPF_ALU64_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "MOV REG32", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU32_IMM(BPF_MOV, R2, 0), - BPF_ALU32_IMM(BPF_MOV, R3, 0), - BPF_ALU32_IMM(BPF_MOV, R4, 0), - BPF_ALU32_IMM(BPF_MOV, R5, 0), - BPF_ALU32_IMM(BPF_MOV, R6, 0), - BPF_ALU32_IMM(BPF_MOV, R7, 0), - BPF_ALU32_IMM(BPF_MOV, R8, 0), - BPF_ALU32_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "LD IMM64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_LD_IMM64(R0, 0x0LL), - BPF_LD_IMM64(R1, 0x0LL), - BPF_LD_IMM64(R2, 0x0LL), - BPF_LD_IMM64(R3, 0x0LL), - BPF_LD_IMM64(R4, 0x0LL), - BPF_LD_IMM64(R5, 0x0LL), - BPF_LD_IMM64(R6, 0x0LL), - BPF_LD_IMM64(R7, 0x0LL), - BPF_LD_IMM64(R8, 0x0LL), - BPF_LD_IMM64(R9, 0x0LL), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { - "INT: ALU MIX", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_ADD, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_XOR, R2, 3), - BPF_ALU64_REG(BPF_DIV, R0, R2), - BPF_JMP_IMM(BPF_JEQ, R0, 10, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOD, R0, 3), - BPF_JMP_IMM(BPF_JEQ, R0, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "INT: shifts by register", - .u.insns_int = { - BPF_MOV64_IMM(R0, -1234), - BPF_MOV64_IMM(R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0x7ffffd97, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R0, R2), - BPF_MOV32_IMM(R4, -1234), - BPF_JMP_REG(BPF_JEQ, R0, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_AND, R4, 63), - BPF_ALU64_REG(BPF_LSH, R0, R4), /* R0 <= 46 */ - BPF_MOV64_IMM(R3, 47), - BPF_ALU64_REG(BPF_ARSH, R0, R3), - BPF_JMP_IMM(BPF_JEQ, R0, -617, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R4, R2), /* R4 = 46 << 1 */ - BPF_JMP_IMM(BPF_JEQ, R4, 92, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 4), - BPF_ALU64_REG(BPF_LSH, R4, R4), /* R4 = 4 << 4 */ - BPF_JMP_IMM(BPF_JEQ, R4, 64, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 5), - BPF_ALU32_REG(BPF_LSH, R4, R4), /* R4 = 5 << 5 */ - BPF_JMP_IMM(BPF_JEQ, R4, 160, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, -#ifdef CONFIG_32BIT - { - "INT: 32-bit context pointer word order and zero-extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JEQ, R1, 0, 3), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_JMP32_IMM(BPF_JNE, R1, 0, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, -#endif - { - "check: missing ret", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: div_k_0", - .u.insns = { - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: unknown insn", - .u.insns = { - /* seccomp insn, rejected in socket filter */ - BPF_STMT(BPF_LDX | BPF_W | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: out of range spill/fill", - .u.insns = { - BPF_STMT(BPF_STX, 16), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "JUMPS + HOLES", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 15), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 3, 4), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC, - { 0x00, 0x1b, 0x21, 0x3c, 0x9d, 0xf8, - 0x90, 0xe2, 0xba, 0x0a, 0x56, 0xb4, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, - 0x20, 0x00, 0x40, 0x11, 0x00, 0x00, /* IP header */ - 0xc0, 0xa8, 0x33, 0x01, - 0xc0, 0xa8, 0x33, 0x02, - 0xbb, 0xb6, - 0xa9, 0xfa, - 0x00, 0x14, 0x00, 0x00, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc }, - { { 88, 0x001b } } - }, - { - "check: RET X", - .u.insns = { - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: LDX + RET X", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 42), - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "M[]: alt STX + LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 100), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 116 } }, - }, - { /* Mainly checking JIT here. */ - "M[]: full STX + full LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0xbadfeedb), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 0xecabedae), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_IMM, 0xafccfeaf), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbffdcedc), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbbbdccb), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbabcbda), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaedecbdb), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_IMM, 0xadebbade), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfcfcfaec), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbcdddbdc), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfeefdfac), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 0xcddcdeea), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaccfaebb), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbdcccdcf), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaaedecde), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfaeacdad), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x2a5a5e5 } }, - }, - { - "check: SKF_AD_MAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MAX), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Passes checker but fails during runtime. */ - "LD [SKF_AD_OFF-1]", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF - 1), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 0 } }, - }, - { - "load 64-bit immediate", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x567800001234LL), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_IMM(BPF_LSH, R3, 32), - BPF_ALU64_IMM(BPF_RSH, R3, 32), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5678, 1), - BPF_EXIT_INSN(), - BPF_JMP_IMM(BPF_JEQ, R3, 0x1234, 1), - BPF_EXIT_INSN(), - BPF_LD_IMM64(R0, 0x1ffffffffLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), /* R0 = 1 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_MOV | BPF_X */ - { - "ALU_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - /* BPF_ALU | BPF_MOV | BPF_K */ - { - "ALU_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_K: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_MOV_K: 0x0000ffffffff0000 = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x00000000ffffffffLL), - BPF_ALU32_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOV_K: small negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU_MOV_K: small negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU_MOV_K: large negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU_MOV_K: large negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_K: dst = 2147483647", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_OR_K: dst = 0x0", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0), - BPF_ALU64_IMM(BPF_MOV, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: dst = -1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: small negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU64_MOV_K: small negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - { - "ALU64_MOV_K: large negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU64_MOV_K: large negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - /* MOVSX32 */ - { - "ALU_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* MOVSX64 REG */ - { - "ALU64_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffdeadbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 32), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_X */ - { - "ALU_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_X: 2 + 4294967294 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_ADD_X: 2 + 4294967294 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_LD_IMM64(R2, 4294967296ULL), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_JMP_REG(BPF_JEQ, R0, R2, 2), - BPF_MOV32_IMM(R0, 0), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_K */ - { - "ALU_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_K: 4294967294 + 2 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_ADD_K: 0 + (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80000000 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80000000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0x80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80008000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 1 + 2147483646 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2147483646), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_ADD_K: 4294967294 + 2 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_LD_IMM64(R1, 4294967296ULL), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_K: 2147483646 + -2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_ADD, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 1 + 0 = 1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x1), - BPF_ALU64_IMM(BPF_ADD, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x80000000 = 0xffffffff80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80000000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0xffffffff80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80008000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_X */ - { - "ALU_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_K */ - { - "ALU_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_SUB_K: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_SUB, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_SUB_K: 4294967294 - 4294967295 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU64_IMM(BPF_SUB, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 2147483646 - 2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_SUB, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MUL | BPF_X */ - { - "ALU_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_X: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0x7FFFFFF8), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_X: -1 * -1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_X: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_X: 64x64 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe5618cf0 } } - }, - { - "ALU64_MUL_X: 64x64 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x2236d88f } } - }, - /* BPF_ALU | BPF_MUL | BPF_K */ - { - "ALU_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_MUL_K: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 0x7FFFFFF8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_K: 1 * (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_MUL_K: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * -2147483647 = -2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 64x32 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe242d208 } } - }, - { - "ALU64_MUL_K: 64x32 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xc28f5c28 } } - }, - /* BPF_ALU | BPF_DIV | BPF_X */ - { - "ALU_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_X: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_X: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R4, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_REG(BPF_DIV, R2, R4), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K */ - { - "ALU_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_DIV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_DIV_K: 0xffffffffffffffff / (-1) = 0x1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x1UL), - BPF_ALU32_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU64_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_DIV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_K: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X */ - { - "ALU_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_X: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967293U), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_X: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483645), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K */ - { - "ALU_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU_MOD_K: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOD, R0, 4294967293U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU64_MOD_K: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_MOD, R0, 2147483645), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_X off=1 (SDIV) */ - { - "ALU_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K off=1 (SDIV) */ - { - "ALU_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_X off=1 (SDIV64) */ - { - "ALU64_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_K off=1 (SDIV64) */ - { - "ALU64_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU64_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X off=1 (SMOD) */ - { - "ALU_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K off=1 (SMOD) */ - { - "ALU_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_X off=1 (SMOD64) */ - { - "ALU64_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ - { - "ALU64_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_AND | BPF_X */ - { - "ALU_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_AND | BPF_K */ - { - "ALU_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_AND_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_AND, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4 } } - }, - { - "ALU_AND_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_AND, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_AND_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0000000080a0c0e0LL), - BPF_ALU32_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU64_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000000000000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000000000000000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0xffffffffffffffff & -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000090b0d0fLL), - BPF_ALU64_IMM(BPF_AND, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456780a0c0e0LL), - BPF_ALU64_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_OR | BPF_X */ - { - "ALU_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_X: 0x0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_X: 0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_OR | BPF_K */ - { - "ALU_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_OR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01020305 } } - }, - { - "ALU_OR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 0xa0b0c0d0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_OR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000f9fbfdffLL), - BPF_ALU32_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU64_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_OR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x000000000000000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x012345678fafcfefLL), - BPF_ALU64_IMM(BPF_OR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfffffffff9fbfdffLL), - BPF_ALU64_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_XOR | BPF_X */ - { - "ALU_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_X: 0x1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_X: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - /* BPF_ALU | BPF_XOR | BPF_K */ - { - "ALU_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU_XOR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_XOR, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x0102030b } } - }, - { - "ALU_XOR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_XOR, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x5e4d3c2b } } - }, - { - "ALU_XOR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000795b3d1fLL), - BPF_ALU32_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU64_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_XOR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ -1 = 0xffff00000000ffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffff00000000ffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x000000000000000 ^ -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456786a4c2e0LL), - BPF_ALU64_IMM(BPF_XOR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfedcba98795b3d1fLL), - BPF_ALU64_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_LSH | BPF_X */ - { - "ALU_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_X: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU64_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } } - }, - /* BPF_ALU | BPF_LSH | BPF_K */ - { - "ALU_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_K: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU_LSH_K: 0x12345678 << 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_RSH | BPF_X */ - { - "ALU_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 20), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU64_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_RSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_RSH | BPF_K */ - { - "ALU_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 20), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU_RSH_K: 0x12345678 >> 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU64_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_ARSH | BPF_X */ - { - "ALU32_ARSH_X: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_ALU32_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU64_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU32_IMM(BPF_MOV, R1, 40), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_ARSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_ARSH | BPF_K */ - { - "ALU32_ARSH_K: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 7), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU32_ARSH_K: -1234 >> 0 = -1234", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1234 } } - }, - { - "ALU64_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU64_IMM(BPF_ARSH, R0, 40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xf123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_NEG */ - { - "ALU_NEG: -(3) = -3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU_NEG: -(-3) = 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_NEG: -(3) = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU64_NEG: -(-3) = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, -3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE */ - { - "ALU_END_FROM_BE 16: 0x0123456789abcdef -> 0xcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0xcdef) } }, - }, - { - "ALU_END_FROM_BE 32: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ - { - "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0x3210) } }, - }, - { - "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x76543210) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE */ - { - "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0xcdef) } }, - }, - { - "ALU_END_FROM_LE 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ - { - "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0x3210) } }, - }, - { - "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x76543210) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BSWAP */ - { - "BSWAP 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcd } }, - }, - { - "BSWAP 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x67452301 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - /* BSWAP, reversed */ - { - "BSWAP 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1032 } }, - }, - { - "BSWAP 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x98badcfe } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 >> 32 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - /* BPF_LDX_MEM B/H/W/DW */ - { - "BPF_LDX_MEM | BPF_B, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_B, R1, R2, -256), - BPF_LDX_MEM(BPF_B, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 256), - BPF_LDX_MEM(BPF_B, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 4096), - BPF_LDX_MEM(BPF_B, R0, R1, 4096), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 4096 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_H, R1, R2, -256), - BPF_LDX_MEM(BPF_H, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 256), - BPF_LDX_MEM(BPF_H, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 8192), - BPF_LDX_MEM(BPF_H, R0, R1, 8192), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 8192 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 13), - BPF_LDX_MEM(BPF_H, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_W, R1, R2, -256), - BPF_LDX_MEM(BPF_W, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 256), - BPF_LDX_MEM(BPF_W, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 16384), - BPF_LDX_MEM(BPF_W, R0, R1, 16384), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 16384 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 13), - BPF_LDX_MEM(BPF_W, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_DW, R1, R2, -256), - BPF_LDX_MEM(BPF_DW, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 256), - BPF_LDX_MEM(BPF_DW, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 32760), - BPF_LDX_MEM(BPF_DW, R0, R1, 32760), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32768, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 13), - BPF_LDX_MEM(BPF_DW, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - /* BPF_LDX_MEMSX B/H/W */ - { - "BPF_LDX_MEMSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead0000000000f0ULL), - BPF_LD_IMM64(R2, 0xfffffffffffffff0ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEMSX(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead00000000f123ULL), - BPF_LD_IMM64(R2, 0xfffffffffffff123ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEMSX(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x00000000deadbeefULL), - BPF_LD_IMM64(R2, 0xffffffffdeadbeefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEMSX(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_STX_MEM B/H/W/DW */ - { - "BPF_STX_MEM | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ - { - "ST_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_B, R10, -40, 0xff), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_B: Store/Load byte: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7f), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7f } }, - .stack_depth = 40, - }, - { - "STX_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffLL), - BPF_STX_MEM(BPF_B, R10, R1, -40), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0xffff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7fff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fff } }, - .stack_depth = 40, - }, - { - "STX_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffLL), - BPF_STX_MEM(BPF_H, R10, R1, -40), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffLL), - BPF_STX_MEM(BPF_W, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative 2", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffff00000000ffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R2, R10, -40), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: first word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x01234567 } }, -#else - { { 0, 0x89abcdef } }, -#endif - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: second word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x89abcdef } }, -#else - { { 0, 0x01234567 } }, -#endif - .stack_depth = 40, - }, - /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ - { - "STX_XADD_W: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxw, - }, - { - "STX_XADD_DW: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxdw, - }, - /* - * Exhaustive tests of atomic operation variants. - * Individual tests are expanded from template macros for all - * combinations of ALU operation, word size and fetching. - */ -#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) - -#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R5, -40), \ - BPF_LDX_MEM(width, R0, R10, -40), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, result } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R1, R10), \ - BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(BPF_W, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R0, -40), \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R1, -40), \ - BPF_ALU64_REG(BPF_SUB, R0, R10), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R3, -40), \ - BPF_ALU32_REG(BPF_MOV, R0, R3), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, (op) & BPF_FETCH ? old : update } }, \ - .stack_depth = 40, \ -} - /* BPF_ATOMIC | BPF_W: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - /* BPF_ATOMIC | BPF_DW: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), -#undef BPF_ATOMIC_POISON -#undef BPF_ATOMIC_OP_TEST1 -#undef BPF_ATOMIC_OP_TEST2 -#undef BPF_ATOMIC_OP_TEST3 -#undef BPF_ATOMIC_OP_TEST4 - /* BPF_ATOMIC | BPF_W, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ALU32_REG(BPF_MOV, R0, R3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - /* BPF_ATOMIC | BPF_DW, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R0, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - /* BPF_JMP32 | BPF_JEQ | BPF_K */ - { - "JMP32_JEQ_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JEQ, R0, 321, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JEQ_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678 & 0xffff, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JEQ_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JEQ | BPF_X */ - { - "JMP32_JEQ_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_K */ - { - "JMP32_JNE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 321, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JNE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678 & 0xffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JNE_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JNE, R0, -123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_X */ - { - "JMP32_JNE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JNE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_K */ - { - "JMP32_JSET_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 2, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 3, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "JMP32_JSET_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x40000000), - BPF_JMP32_IMM(BPF_JSET, R0, 0x3fffffff, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 0x60000000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x40000000 } } - }, - { - "JMP32_JSET_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSET, R0, -1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_X */ - { - "JMP32_JSET_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 8), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_JMP32_REG(BPF_JSET, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 8 | 2), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 8 } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_K */ - { - "JMP32_JGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGT, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 0xfffffffd, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_X */ - { - "JMP32_JGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_K */ - { - "JMP32_JGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGE, R0, 124, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGE, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_X */ - { - "JMP32_JGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_K */ - { - "JMP32_JLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLT, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 0xffffffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_X */ - { - "JMP32_JLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_K */ - { - "JMP32_JLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLE, R0, 122, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_X */ - { - "JMP32_JLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_K */ - { - "JMP32_JSGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345679, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_X */ - { - "JMP32_JSGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_K */ - { - "JMP32_JSGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGE, R0, -122, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345677, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_X */ - { - "JMP32_JSGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_K */ - { - "JMP32_JSLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345677, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_X */ - { - "JMP32_JSLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLE, R0, -124, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345679, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP | BPF_EXIT */ - { - "JMP_EXIT", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x4711), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0x4712), - }, - INTERNAL, - { }, - { { 0, 0x4711 } }, - }, - /* BPF_JMP | BPF_JA */ - { - "JMP_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JA, 0, 0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP32 | BPF_JA */ - { - "JMP32_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JA, 0, 1, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_K */ - { - "JMP_JSLT_K: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_K: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_K */ - { - "JMP_JSGT_K: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_K: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_K */ - { - "JMP_JSLE_K: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 6), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_K */ - { - "JMP_JSGE_K: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 6), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K */ - { - "JMP_JGT_K: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGT, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_JMP_IMM(BPF_JGT, R1, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K */ - { - "JMP_JLT_K: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLT, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 1), - BPF_JMP_IMM(BPF_JLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_K */ - { - "JMP_JGE_K: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_K */ - { - "JMP_JLE_K: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (3 > 2) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 3), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JGT, R1, 2, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_K: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (2 < 3) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 2), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JLT, R1, 3, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_K: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_K */ - { - "JMP_JNE_K: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JNE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_K */ - { - "JMP_JEQ_K: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JEQ, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_K */ - { - "JMP_JSET_K: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_K: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 0xffffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_X */ - { - "JMP_JSGT_X: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_X: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_X */ - { - "JMP_JSLT_X: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_X: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_X */ - { - "JMP_JSGE_X: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_X */ - { - "JMP_JSLE_X: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_X */ - { - "JMP_JGT_X: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_X: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_X */ - { - "JMP_JLT_X: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLT_X: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_X */ - { - "JMP_JGE_X: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_X */ - { - "JMP_JLE_X: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - /* Mainly testing JIT + imm64 here. */ - "JMP_JGE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JGE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JGE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JLE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JLE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_X */ - { - "JMP_JNE_X: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JNE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_X */ - { - "JMP_JEQ_X: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_X */ - { - "JMP_JSET_X: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_X: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 0xffffffff), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JA: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_ja, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Maximum possible literals", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns1, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Single literal", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfefefefe } }, - .fill_helper = bpf_fill_maxinsns2, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Run/add until end", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x947bf368 } }, - .fill_helper = bpf_fill_maxinsns3, - }, - { - "BPF_MAXINSNS: Too many instructions", - { }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = bpf_fill_maxinsns4, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns5, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Ctx heavy transformations", - { }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - .fill_helper = bpf_fill_maxinsns6, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Call heavy transformations", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 1, 0 }, { 10, 0 } }, - .fill_helper = bpf_fill_maxinsns7, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Jump heavy test", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns8, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump backwards", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xcbababab } }, - .fill_helper = bpf_fill_maxinsns9, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Edge hopping nuthouse", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xabababac } }, - .fill_helper = bpf_fill_maxinsns10, - }, - { - "BPF_MAXINSNS: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_maxinsns11, - }, - { - "BPF_MAXINSNS: jump over MSH", - { }, - CLASSIC | FLAG_EXPECTED_FAIL, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns12, - .expected_errcode = -EINVAL, - }, - { - "BPF_MAXINSNS: exec all MSH", - { }, - CLASSIC, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xababab83 } }, - .fill_helper = bpf_fill_maxinsns13, - }, - { - "BPF_MAXINSNS: ld_abs+get_processor_id", - { }, - CLASSIC, - { }, - { { 1, 0xbee } }, - .fill_helper = bpf_fill_ld_abs_get_processor_id, - }, - /* - * LD_IND / LD_ABS on fragmented SKBs - */ - { - "LD_IND byte frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND word frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x8), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_IND word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS byte frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x40), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x44), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS word frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x48), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - /* - * LD_IND / LD_ABS on non fragmented SKBs - */ - { - /* - * this tests that the JIT/interpreter correctly resets X - * before using it in an LD_IND instruction. - */ - "LD_IND byte default X", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x1] = 0x42 }, - { {0x40, 0x42 } }, - }, - { - "LD_IND byte positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 } }, - }, - { - "LD_IND byte negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x05 } }, - }, - { - "LD_IND byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_IND byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 1), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 2), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 3), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_IND halfword positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_IND halfword negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xbb66 } }, - }, - { - "LD_IND halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0x66cc } }, - }, - { - "LD_IND halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3d), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_IND halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND word positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xee99ffaa } }, - }, - { - "LD_IND word negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xbb66cc77 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x55bb66cc } }, - }, - { - "LD_IND word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x66cc77dd } }, - }, - { - "LD_IND word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_IND word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_ABS byte", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x20), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xcc } }, - }, - { - "LD_ABS byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_ABS byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS byte negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3d), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS halfword", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_ABS halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x25), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x99ff } }, - }, - { - "LD_ABS halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_ABS halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS halfword negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x1982 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x1c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88ee99 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x21), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x77dd88ee } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x23), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x88ee99ff } }, - }, - { - "LD_ABS word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_ABS word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS word negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x25051982 }, }, - }, - { - "LD_ABS word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LDX_MSH standalone, preserved A", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0xffeebbaa }, }, - }, - { - "LDX_MSH standalone, preserved A 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0x175e9d63), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3d), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x175e9d63 }, }, - }, - { - "LDX_MSH standalone, test result 1", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x14 }, }, - }, - { - "LDX_MSH standalone, test result 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, negative offset", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, -1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - { - "LDX_MSH standalone, negative offset 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x40), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - /* - * verify that the interpreter or JIT correctly sets A and X - * to 0. - */ - { - "ADD default X", - .u.insns = { - /* - * A = 0x42 - * A = A + X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "ADD default A", - .u.insns = { - /* - * A = A + 0x42 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "SUB default X", - .u.insns = { - /* - * A = 0x66 - * A = A - X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x66), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "SUB default A", - .u.insns = { - /* - * A = A - -0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, -0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "MUL default X", - .u.insns = { - /* - * A = 0x42 - * A = A * X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MUL default A", - .u.insns = { - /* - * A = A * 0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default X", - .u.insns = { - /* - * A = 0x42 - * A = A / X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default A", - .u.insns = { - /* - * A = A / 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default X", - .u.insns = { - /* - * A = 0x42 - * A = A mod X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default A", - .u.insns = { - /* - * A = A mod 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "JMP EQ default A", - .u.insns = { - /* - * cmp A, 0x0, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "JMP EQ default X", - .u.insns = { - /* - * A = 0x0 - * cmp A, X, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - /* Checking interpreter vs JIT wrt signed extended imms. */ - { - "JNE signed compare, test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_ALU32_IMM(BPF_MOV, R4, 0xfefb0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R2, R4, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 4", - .u.insns_int = { - BPF_LD_IMM64(R1, -17104896), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 5", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xfefb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 6", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x7efb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0x7efb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 7", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffff0000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xfefbbc12), - BPF_STMT(BPF_ALU | BPF_AND | BPF_X, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0xfefb0000, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, 2), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { { 0, 2 } }, - }, - /* BPF_LDX_MEM with operand aliasing */ - { - "LDX_MEM_B: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_B, R10, -8, 123), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_B, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_H: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_H, R10, -8, 12345), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_H, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_W: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -8, 123456789), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_W, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123456789 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_DW: operand register aliasing", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x123456789abcdefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_DW, R0, R0, -8), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_MOV64_REG(R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* - * Register (non-)clobbering tests for the case where a JIT implements - * complex ALU or ATOMIC operations via function calls. If so, the - * function call must be transparent to the eBPF registers. The JIT - * must therefore save and restore relevant registers across the call. - * The following tests check that the eBPF registers retain their - * values after such an operation. Mainly intended for complex ALU - * and atomic operation, but we run it for all. You never know... - * - * Note that each operations should be tested twice with different - * destinations, to check preservation for all registers. - */ -#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ - { \ - #alu "_" #op " to " #dst ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, R0), \ - BPF_ALU64_IMM(BPF_MOV, R1, R1), \ - BPF_ALU64_IMM(BPF_MOV, R2, R2), \ - BPF_ALU64_IMM(BPF_MOV, R3, R3), \ - BPF_ALU64_IMM(BPF_MOV, R4, R4), \ - BPF_ALU64_IMM(BPF_MOV, R5, R5), \ - BPF_ALU64_IMM(BPF_MOV, R6, R6), \ - BPF_ALU64_IMM(BPF_MOV, R7, R7), \ - BPF_ALU64_IMM(BPF_MOV, R8, R8), \ - BPF_ALU64_IMM(BPF_MOV, R9, R9), \ - BPF_##alu(BPF_ ##op, dst, src), \ - BPF_ALU32_IMM(BPF_MOV, dst, dst), \ - BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } } \ - } - /* ALU64 operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), - /* ALU32 immediate operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), - /* ALU64 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), - /* ALU32 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), -#undef BPF_TEST_CLOBBER_ALU -#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ - { \ - "Atomic_" #width " " #op ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, 0), \ - BPF_ALU64_IMM(BPF_MOV, R1, 1), \ - BPF_ALU64_IMM(BPF_MOV, R2, 2), \ - BPF_ALU64_IMM(BPF_MOV, R3, 3), \ - BPF_ALU64_IMM(BPF_MOV, R4, 4), \ - BPF_ALU64_IMM(BPF_MOV, R5, 5), \ - BPF_ALU64_IMM(BPF_MOV, R6, 6), \ - BPF_ALU64_IMM(BPF_MOV, R7, 7), \ - BPF_ALU64_IMM(BPF_MOV, R8, 8), \ - BPF_ALU64_IMM(BPF_MOV, R9, 9), \ - BPF_ST_MEM(width, R10, -8, \ - (op) == BPF_CMPXCHG ? 0 : \ - (op) & BPF_FETCH ? 1 : 0), \ - BPF_ATOMIC_OP(width, op, R10, R1, -8), \ - BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } }, \ - .stack_depth = 8, \ - } - /* 64-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), - /* 32-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), -#undef BPF_TEST_CLOBBER_ATOMIC - /* Checking that ALU32 src is not zero extended in place */ -#define BPF_ALU32_SRC_ZEXT(op) \ - { \ - "ALU32_" #op "_X: src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R0, R1), \ - BPF_ALU32_REG(BPF_##op, R2, R1), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_ALU32_SRC_ZEXT(MOV), - BPF_ALU32_SRC_ZEXT(AND), - BPF_ALU32_SRC_ZEXT(OR), - BPF_ALU32_SRC_ZEXT(XOR), - BPF_ALU32_SRC_ZEXT(ADD), - BPF_ALU32_SRC_ZEXT(SUB), - BPF_ALU32_SRC_ZEXT(MUL), - BPF_ALU32_SRC_ZEXT(DIV), - BPF_ALU32_SRC_ZEXT(MOD), -#undef BPF_ALU32_SRC_ZEXT - /* Checking that ATOMIC32 src is not zero extended in place */ -#define BPF_ATOMIC32_SRC_ZEXT(op) \ - { \ - "ATOMIC_W_" #op ": src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ST_MEM(BPF_W, R10, -4, 0), \ - BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 8, \ - } - BPF_ATOMIC32_SRC_ZEXT(ADD), - BPF_ATOMIC32_SRC_ZEXT(AND), - BPF_ATOMIC32_SRC_ZEXT(OR), - BPF_ATOMIC32_SRC_ZEXT(XOR), -#undef BPF_ATOMIC32_SRC_ZEXT - /* Checking that CMPXCHG32 src is not zero extended in place */ - { - "ATOMIC_W_CMPXCHG: src preserved in zext", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789acbdefULL), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_REG(BPF_MOV, R0, 0), - BPF_ST_MEM(BPF_W, R10, -4, 0), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_OR, R1, R2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* Checking that JMP32 immediate src is not zero extended in place */ -#define BPF_JMP32_IMM_ZEXT(op) \ - { \ - "JMP32_" #op "_K: operand preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_IMM_ZEXT(JEQ), - BPF_JMP32_IMM_ZEXT(JNE), - BPF_JMP32_IMM_ZEXT(JSET), - BPF_JMP32_IMM_ZEXT(JGT), - BPF_JMP32_IMM_ZEXT(JGE), - BPF_JMP32_IMM_ZEXT(JLT), - BPF_JMP32_IMM_ZEXT(JLE), - BPF_JMP32_IMM_ZEXT(JSGT), - BPF_JMP32_IMM_ZEXT(JSGE), - BPF_JMP32_IMM_ZEXT(JSLT), - BPF_JMP32_IMM_ZEXT(JSLE), -#undef BPF_JMP2_IMM_ZEXT - /* Checking that JMP32 dst & src are not zero extended in place */ -#define BPF_JMP32_REG_ZEXT(op) \ - { \ - "JMP32_" #op "_X: operands preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R2, R0), \ - BPF_ALU64_REG(BPF_MOV, R3, R1), \ - BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R2), \ - BPF_ALU64_REG(BPF_SUB, R1, R3), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_REG_ZEXT(JEQ), - BPF_JMP32_REG_ZEXT(JNE), - BPF_JMP32_REG_ZEXT(JSET), - BPF_JMP32_REG_ZEXT(JGT), - BPF_JMP32_REG_ZEXT(JGE), - BPF_JMP32_REG_ZEXT(JLT), - BPF_JMP32_REG_ZEXT(JLE), - BPF_JMP32_REG_ZEXT(JSGT), - BPF_JMP32_REG_ZEXT(JSGE), - BPF_JMP32_REG_ZEXT(JSLT), - BPF_JMP32_REG_ZEXT(JSLE), -#undef BPF_JMP2_REG_ZEXT - /* ALU64 K register combinations */ - { - "ALU64_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm_regs, - }, - { - "ALU64_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm_regs, - }, - { - "ALU64_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm_regs, - }, - { - "ALU64_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm_regs, - }, - { - "ALU64_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm_regs, - }, - { - "ALU64_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm_regs, - }, - { - "ALU64_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm_regs, - }, - { - "ALU64_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm_regs, - }, - { - "ALU64_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm_regs, - }, - { - "ALU64_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm_regs, - }, - { - "ALU64_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm_regs, - }, - { - "ALU64_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm_regs, - }, - { - "ALU64_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm_regs, - }, - { - "ALU64_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm_regs, - }, - /* ALU32 K registers */ - { - "ALU32_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm_regs, - }, - { - "ALU32_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm_regs, - }, - { - "ALU32_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm_regs, - }, - { - "ALU32_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm_regs, - }, - { - "ALU32_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm_regs, - }, - { - "ALU32_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm_regs, - }, - { - "ALU32_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm_regs, - }, - { - "ALU32_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm_regs, - }, - { - "ALU32_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm_regs, - }, - { - "ALU32_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm_regs, - }, - { - "ALU32_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm_regs, - }, - { - "ALU32_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm_regs, - }, - { - "ALU32_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm_regs, - }, - { - "ALU32_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm_regs, - }, - /* ALU64 X register combinations */ - { - "ALU64_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg_pairs, - }, - { - "ALU64_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg_pairs, - }, - { - "ALU64_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg_pairs, - }, - { - "ALU64_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg_pairs, - }, - { - "ALU64_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg_pairs, - }, - { - "ALU64_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg_pairs, - }, - { - "ALU64_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg_pairs, - }, - { - "ALU64_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg_pairs, - }, - { - "ALU64_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg_pairs, - }, - { - "ALU64_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg_pairs, - }, - { - "ALU64_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg_pairs, - }, - { - "ALU64_MOD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg_pairs, - }, - /* ALU32 X register combinations */ - { - "ALU32_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg_pairs, - }, - { - "ALU32_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg_pairs, - }, - { - "ALU32_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg_pairs, - }, - { - "ALU32_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg_pairs, - }, - { - "ALU32_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg_pairs, - }, - { - "ALU32_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg_pairs, - }, - { - "ALU32_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg_pairs, - }, - { - "ALU32_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg_pairs, - }, - { - "ALU32_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg_pairs, - }, - { - "ALU32_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg_pairs, - }, - { - "ALU32_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg_pairs, - }, - { - "ALU32_MOD_X register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg_pairs, - }, - /* Exhaustive test of ALU64 shift operations */ - { - "ALU64_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm, - }, - { - "ALU64_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm, - }, - { - "ALU64_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm, - }, - { - "ALU64_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg, - }, - { - "ALU64_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg, - }, - { - "ALU64_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg, - }, - /* Exhaustive test of ALU32 shift operations */ - { - "ALU32_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm, - }, - { - "ALU32_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm, - }, - { - "ALU32_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm, - }, - { - "ALU32_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg, - }, - { - "ALU32_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg, - }, - { - "ALU32_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg, - }, - /* - * Exhaustive test of ALU64 shift operations when - * source and destination register are the same. - */ - { - "ALU64_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_same_reg, - }, - { - "ALU64_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_same_reg, - }, - { - "ALU64_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_same_reg, - }, - /* - * Exhaustive test of ALU32 shift operations when - * source and destination register are the same. - */ - { - "ALU32_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_same_reg, - }, - { - "ALU32_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_same_reg, - }, - { - "ALU32_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_same_reg, - }, - /* ALU64 immediate magnitudes */ - { - "ALU64_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 immediate magnitudes */ - { - "ALU32_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU64 register magnitudes */ - { - "ALU64_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 register magnitudes */ - { - "ALU32_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* LD_IMM64 immediate magnitudes and byte patterns */ - { - "LD_IMM64: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_magn, - }, - { - "LD_IMM64: checker byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_checker, - }, - { - "LD_IMM64: random positive and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_zero, - }, - { - "LD_IMM64: random negative and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_neg_zero, - }, - { - "LD_IMM64: random positive and negative byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_neg, - }, - /* 64-bit ATOMIC register combinations */ - { - "ATOMIC_DW_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 32-bit ATOMIC register combinations */ - { - "ATOMIC_W_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 64-bit ATOMIC magnitudes */ - { - "ATOMIC_DW_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg64, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* 32-bit atomic magnitudes */ - { - "ATOMIC_W_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg32, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP immediate magnitudes */ - { - "JMP_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP register magnitudes */ - { - "JMP_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 immediate magnitudes */ - { - "JMP32_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 register magnitudes */ - { - "JMP32_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* Conditional jumps with constant decision */ - { - "JMP_JSET_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JSET, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLT, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGE_K: imm = 0 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGE, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: imm = 0xffffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLE_K: imm = 0xffffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSGE_K: imm = -0x80000000 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSLT_K: imm = -0x80000000 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JEQ_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JEQ, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JNE_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - /* Short relative jumps */ - { - "Short relative jump: offset=0", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=1", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=3", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=4", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - /* Conditional branch conversions */ - { - "Long conditional jump: taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken_32, - }, - { - "Long conditional jump: not taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken_32, - }, - { - "Long conditional jump: always taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken_32, - }, - { - "Long conditional jump: never taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken_32, - }, - { - "Long conditional jump: taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken, - }, - { - "Long conditional jump: not taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken, - }, - { - "Long conditional jump: always taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken, - }, - { - "Long conditional jump: never taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken, - }, - /* Staggered jump sequences, immediate */ - { - "Staggered jumps: JMP_JA", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_ja, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, register */ - { - "Staggered jumps: JMP_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 immediate */ - { - "Staggered jumps: JMP32_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 register */ - { - "Staggered jumps: JMP32_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, -}; - -static struct net_device dev; - -static struct sk_buff *populate_skb(char *buf, int size) -{ - struct sk_buff *skb; - - if (size >= MAX_DATA) - return NULL; - - skb = alloc_skb(MAX_DATA, GFP_KERNEL); - if (!skb) - return NULL; - - __skb_put_data(skb, buf, size); - - /* Initialize a fake skb with test pattern. */ - skb_reset_mac_header(skb); - skb->protocol = htons(ETH_P_IP); - skb->pkt_type = SKB_TYPE; - skb->mark = SKB_MARK; - skb->hash = SKB_HASH; - skb->queue_mapping = SKB_QUEUE_MAP; - skb->vlan_tci = SKB_VLAN_TCI; - skb->vlan_proto = htons(ETH_P_IP); - dev_net_set(&dev, &init_net); - skb->dev = &dev; - skb->dev->ifindex = SKB_DEV_IFINDEX; - skb->dev->type = SKB_DEV_TYPE; - skb_set_network_header(skb, min(size, ETH_HLEN)); - - return skb; -} - -static void *generate_test_data(struct bpf_test *test, int sub) -{ - struct sk_buff *skb; - struct page *page; - - if (test->aux & FLAG_NO_DATA) - return NULL; - - if (test->aux & FLAG_LARGE_MEM) - return kmalloc(test->test[sub].data_size, GFP_KERNEL); - - /* Test case expects an skb, so populate one. Various - * subtests generate skbs of different sizes based on - * the same data. - */ - skb = populate_skb(test->data, test->test[sub].data_size); - if (!skb) - return NULL; - - if (test->aux & FLAG_SKB_FRAG) { - /* - * when the test requires a fragmented skb, add a - * single fragment to the skb, filled with - * test->frag_data. - */ - page = alloc_page(GFP_KERNEL); - if (!page) - goto err_kfree_skb; - - memcpy(page_address(page), test->frag_data, MAX_DATA); - skb_add_rx_frag(skb, 0, page, 0, MAX_DATA, MAX_DATA); - } - - return skb; -err_kfree_skb: - kfree_skb(skb); - return NULL; -} - -static void release_test_data(const struct bpf_test *test, void *data) -{ - if (test->aux & FLAG_NO_DATA) - return; - - if (test->aux & FLAG_LARGE_MEM) - kfree(data); - else - kfree_skb(data); -} - -static int filter_length(int which) -{ - struct sock_filter *fp; - int len; - - if (tests[which].fill_helper) - return tests[which].u.ptr.len; - - fp = tests[which].u.insns; - for (len = MAX_INSNS - 1; len > 0; --len) - if (fp[len].code != 0 || fp[len].k != 0) - break; - - return len + 1; -} - -static void *filter_pointer(int which) -{ - if (tests[which].fill_helper) - return tests[which].u.ptr.insns; - else - return tests[which].u.insns; -} - -static struct bpf_prog *generate_filter(int which, int *err) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - unsigned int flen = filter_length(which); - void *fptr = filter_pointer(which); - struct sock_fprog_kern fprog; - struct bpf_prog *fp; - - switch (test_type) { - case CLASSIC: - fprog.filter = fptr; - fprog.len = flen; - - *err = bpf_prog_create(&fp, &fprog); - if (tests[which].aux & FLAG_EXPECTED_FAIL) { - if (*err == tests[which].expected_errcode) { - pr_cont("PASS\n"); - /* Verifier rejected filter as expected. */ - *err = 0; - return NULL; - } else { - pr_cont("UNEXPECTED_PASS\n"); - /* Verifier didn't reject the test that's - * bad enough, just return! - */ - *err = -EINVAL; - return NULL; - } - } - if (*err) { - pr_cont("FAIL to prog_create err=%d len=%d\n", - *err, fprog.len); - return NULL; - } - break; - - case INTERNAL: - fp = bpf_prog_alloc(bpf_prog_size(flen), 0); - if (fp == NULL) { - pr_cont("UNEXPECTED_FAIL no memory left\n"); - *err = -ENOMEM; - return NULL; - } - - fp->len = flen; - /* Type doesn't really matter here as long as it's not unspec. */ - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); - fp->aux->stack_depth = tests[which].stack_depth; - fp->aux->verifier_zext = !!(tests[which].aux & - FLAG_VERIFIER_ZEXT); - - /* We cannot error here as we don't need type compatibility - * checks. - */ - fp = bpf_prog_select_runtime(fp, err); - if (*err) { - pr_cont("FAIL to select_runtime err=%d\n", *err); - return NULL; - } - break; - } - - *err = 0; - return fp; -} - -static void release_filter(struct bpf_prog *fp, int which) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - - switch (test_type) { - case CLASSIC: - bpf_prog_destroy(fp); - break; - case INTERNAL: - bpf_prog_free(fp); - break; - } -} - -static int __run_one(const struct bpf_prog *fp, const void *data, - int runs, u64 *duration) -{ - u64 start, finish; - int ret = 0, i; - - migrate_disable(); - start = ktime_get_ns(); - - for (i = 0; i < runs; i++) - ret = bpf_prog_run(fp, data); - - finish = ktime_get_ns(); - migrate_enable(); - - *duration = finish - start; - do_div(*duration, runs); - - return ret; -} - -static int run_one(const struct bpf_prog *fp, struct bpf_test *test) -{ - int err_cnt = 0, i, runs = MAX_TESTRUNS; - - if (test->nr_testruns) - runs = min(test->nr_testruns, MAX_TESTRUNS); - - for (i = 0; i < MAX_SUBTESTS; i++) { - void *data; - u64 duration; - u32 ret; - - /* - * NOTE: Several sub-tests may be present, in which case - * a zero {data_size, result} tuple indicates the end of - * the sub-test array. The first test is always run, - * even if both data_size and result happen to be zero. - */ - if (i > 0 && - test->test[i].data_size == 0 && - test->test[i].result == 0) - break; - - data = generate_test_data(test, i); - if (!data && !(test->aux & FLAG_NO_DATA)) { - pr_cont("data generation failed "); - err_cnt++; - break; - } - ret = __run_one(fp, data, runs, &duration); - release_test_data(test, data); - - if (ret == test->test[i].result) { - pr_cont("%lld ", duration); - } else { - s32 res = test->test[i].result; - - pr_cont("ret %d != %d (%#x != %#x)", - ret, res, ret, res); - err_cnt++; - } - } - - return err_cnt; -} - -static char test_name[64]; -module_param_string(test_name, test_name, sizeof(test_name), 0); - -static int test_id = -1; -module_param(test_id, int, 0); - -static int test_range[2] = { 0, INT_MAX }; -module_param_array(test_range, int, NULL, 0); - -static bool exclude_test(int test_id) -{ - return test_id < test_range[0] || test_id > test_range[1]; -} - -static __init struct sk_buff *build_test_skb(void) -{ - u32 headroom = NET_SKB_PAD + NET_IP_ALIGN + ETH_HLEN; - struct sk_buff *skb[2]; - struct page *page[2]; - int i, data_size = 8; - - for (i = 0; i < 2; i++) { - page[i] = alloc_page(GFP_KERNEL); - if (!page[i]) { - if (i == 0) - goto err_page0; - else - goto err_page1; - } - - /* this will set skb[i]->head_frag */ - skb[i] = dev_alloc_skb(headroom + data_size); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb_reserve(skb[i], headroom); - skb_put(skb[i], data_size); - skb[i]->protocol = htons(ETH_P_IP); - skb_reset_network_header(skb[i]); - skb_set_mac_header(skb[i], -ETH_HLEN); - - skb_add_rx_frag(skb[i], 0, page[i], 0, 64, 64); - // skb_headlen(skb[i]): 8, skb[i]->head_frag = 1 - } - - /* setup shinfo */ - skb_shinfo(skb[0])->gso_size = 1448; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV4; - skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->data_len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - __free_page(page[1]); -err_page1: - kfree_skb(skb[0]); -err_skb0: - __free_page(page[0]); -err_page0: - return NULL; -} - -static __init struct sk_buff *build_test_skb_linear_no_head_frag(void) -{ - unsigned int alloc_size = 2000; - unsigned int headroom = 102, doffset = 72, data_size = 1308; - struct sk_buff *skb[2]; - int i; - - /* skbs linked in a frag_list, both with linear data, with head_frag=0 - * (data allocated by kmalloc), both have tcp data of 1308 bytes - * (total payload is 2616 bytes). - * Data offset is 72 bytes (40 ipv6 hdr, 32 tcp hdr). Some headroom. - */ - for (i = 0; i < 2; i++) { - skb[i] = alloc_skb(alloc_size, GFP_KERNEL); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb[i]->protocol = htons(ETH_P_IPV6); - skb_reserve(skb[i], headroom); - skb_put(skb[i], doffset + data_size); - skb_reset_network_header(skb[i]); - if (i == 0) - skb_reset_mac_header(skb[i]); - else - skb_set_mac_header(skb[i], -ETH_HLEN); - __skb_pull(skb[i], doffset); - } - - /* setup shinfo. - * mimic bpf_skb_proto_4_to_6, which resets gso_segs and assigns a - * reduced gso_size. - */ - skb_shinfo(skb[0])->gso_size = 1288; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV6 | SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - kfree_skb(skb[0]); -err_skb0: - return NULL; -} - -struct skb_segment_test { - const char *descr; - struct sk_buff *(*build_skb)(void); - netdev_features_t features; -}; - -static struct skb_segment_test skb_segment_tests[] __initconst = { - { - .descr = "gso_with_rx_frags", - .build_skb = build_test_skb, - .features = NETIF_F_SG | NETIF_F_GSO_PARTIAL | NETIF_F_IP_CSUM | - NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM - }, - { - .descr = "gso_linear_no_head_frag", - .build_skb = build_test_skb_linear_no_head_frag, - .features = NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_GSO | - NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | - NETIF_F_HW_VLAN_STAG_TX - } -}; - -static __init int test_skb_segment_single(const struct skb_segment_test *test) -{ - struct sk_buff *skb, *segs; - int ret = -1; - - skb = test->build_skb(); - if (!skb) { - pr_info("%s: failed to build_test_skb", __func__); - goto done; - } - - segs = skb_segment(skb, test->features); - if (!IS_ERR(segs)) { - kfree_skb_list(segs); - ret = 0; - } - kfree_skb(skb); -done: - return ret; -} - -static __init int test_skb_segment(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - const struct skb_segment_test *test = &skb_segment_tests[i]; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - - if (test_skb_segment_single(test)) { - pr_cont("FAIL\n"); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED\n", __func__, - pass_cnt, err_cnt); - return err_cnt ? -EINVAL : 0; -} - -static __init int test_bpf(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - struct bpf_prog *fp; - int err; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, tests[i].descr); - - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) { - pr_cont("FAIL to prog_fill\n"); - continue; - } - - fp = generate_filter(i, &err); - - if (tests[i].fill_helper) { - kfree(tests[i].u.ptr.insns); - tests[i].u.ptr.insns = NULL; - } - - if (fp == NULL) { - if (err == 0) { - pass_cnt++; - continue; - } - err_cnt++; - continue; - } - - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - err = run_one(fp, &tests[i]); - release_filter(fp, i); - - if (err) { - pr_cont("FAIL (%d times)\n", err); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -struct tail_call_test { - const char *descr; - struct bpf_insn insns[MAX_INSNS]; - int flags; - int result; - int stack_depth; - bool has_tail_call; -}; - -/* Flags that can be passed to tail call test cases */ -#define FLAG_NEED_STATE BIT(0) -#define FLAG_RESULT_IN_STATE BIT(1) - -/* - * Magic marker used in test snippets for tail calls below. - * BPF_LD/MOV to R2 and R2 with this immediate value is replaced - * with the proper values by the test runner. - */ -#define TAIL_CALL_MARKER 0x7a11ca11 - -/* Special offset to indicate a NULL call target */ -#define TAIL_CALL_NULL 0x7fff - -/* Special offset to indicate an out-of-range index */ -#define TAIL_CALL_INVALID 0x7ffe - -#define TAIL_CALL(offset) \ - BPF_LD_IMM64(R2, TAIL_CALL_MARKER), \ - BPF_RAW_INSN(BPF_ALU | BPF_MOV | BPF_K, R3, 0, \ - offset, TAIL_CALL_MARKER), \ - BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) - -/* - * A test function to be called from a BPF program, clobbering a lot of - * CPU registers in the process. A JITed BPF program calling this function - * must save and restore any caller-saved registers it uses for internal - * state, for example the current tail call count. - */ -BPF_CALL_1(bpf_test_func, u64, arg) -{ - char buf[64]; - long a = 0; - long b = 1; - long c = 2; - long d = 3; - long e = 4; - long f = 5; - long g = 6; - long h = 7; - - return snprintf(buf, sizeof(buf), - "%ld %lu %lx %ld %lu %lx %ld %lu %x", - a, b, c, d, e, f, g, h, (int)arg); -} -#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID - -/* - * Tail call tests. Each test case may call any other test in the table, - * including itself, specified as a relative index offset from the calling - * test. The index TAIL_CALL_NULL can be used to specify a NULL target - * function to test the JIT error path. Similarly, the index TAIL_CALL_INVALID - * results in a target index that is out of range. - */ -static struct tail_call_test tail_call_tests[] = { - { - "Tail call leaf", - .insns = { - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - }, - .result = 1, - }, - { - "Tail call 2", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 2), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 3, - .has_tail_call = true, - }, - { - "Tail call 3", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 3), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 6, - .has_tail_call = true, - }, - { - "Tail call 4", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 4), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 10, - .has_tail_call = true, - }, - { - "Tail call load/store leaf", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_REG(BPF_MOV, R3, BPF_REG_FP), - BPF_STX_MEM(BPF_DW, R3, R1, -8), - BPF_STX_MEM(BPF_DW, R3, R2, -16), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 3), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -16), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 32, - }, - { - "Tail call load/store", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R0, 3), - BPF_STX_MEM(BPF_DW, BPF_REG_FP, R0, -8), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 16, - .has_tail_call = true, - }, - { - "Tail call error path, max count reached", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call count preserved across function calls", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_CALL_REL(BPF_FUNC_get_numa_node_id), - BPF_CALL_REL(BPF_FUNC_ktime_get_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), - BPF_CALL_REL(BPF_FUNC_jiffies64), - BPF_CALL_REL(BPF_FUNC_test_func), - BPF_LDX_MEM(BPF_DW, R1, R10, -8), - BPF_ALU32_REG(BPF_MOV, R0, R1), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .stack_depth = 8, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, NULL target", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_NULL), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, index out of range", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_INVALID), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, -}; - -static void __init destroy_tail_call_tests(struct bpf_array *progs) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) - if (progs->ptrs[i]) - bpf_prog_free(progs->ptrs[i]); - kfree(progs); -} - -static __init int prepare_tail_call_tests(struct bpf_array **pprogs) -{ - int ntests = ARRAY_SIZE(tail_call_tests); - struct bpf_array *progs; - int which, err; - - /* Allocate the table of programs to be used for tail calls */ - progs = kzalloc_flex(*progs, ptrs, ntests + 1); - if (!progs) - goto out_nomem; - - /* Create all eBPF programs and populate the table */ - for (which = 0; which < ntests; which++) { - struct tail_call_test *test = &tail_call_tests[which]; - struct bpf_prog *fp; - int len, i; - - /* Compute the number of program instructions */ - for (len = 0; len < MAX_INSNS; len++) { - struct bpf_insn *insn = &test->insns[len]; - - if (len < MAX_INSNS - 1 && - insn->code == (BPF_LD | BPF_DW | BPF_IMM)) - len++; - if (insn->code == 0) - break; - } - - /* Allocate and initialize the program */ - fp = bpf_prog_alloc(bpf_prog_size(len), 0); - if (!fp) - goto out_nomem; - - fp->len = len; - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - fp->aux->stack_depth = test->stack_depth; - fp->aux->tail_call_reachable = test->has_tail_call; - memcpy(fp->insnsi, test->insns, len * sizeof(struct bpf_insn)); - - /* Relocate runtime tail call offsets and addresses */ - for (i = 0; i < len; i++) { - struct bpf_insn *insn = &fp->insnsi[i]; - long addr = 0; - - switch (insn->code) { - case BPF_LD | BPF_DW | BPF_IMM: - if (insn->imm != TAIL_CALL_MARKER) - break; - insn[0].imm = (u32)(long)progs; - insn[1].imm = ((u64)(long)progs) >> 32; - break; - - case BPF_ALU | BPF_MOV | BPF_K: - if (insn->imm != TAIL_CALL_MARKER) - break; - if (insn->off == TAIL_CALL_NULL) - insn->imm = ntests; - else if (insn->off == TAIL_CALL_INVALID) - insn->imm = ntests + 1; - else - insn->imm = which + insn->off; - insn->off = 0; - break; - - case BPF_JMP | BPF_CALL: - if (insn->src_reg != BPF_PSEUDO_CALL) - break; - switch (insn->imm) { - case BPF_FUNC_get_numa_node_id: - addr = (long)&numa_node_id; - break; - case BPF_FUNC_ktime_get_ns: - addr = (long)&ktime_get_ns; - break; - case BPF_FUNC_ktime_get_boot_ns: - addr = (long)&ktime_get_boot_fast_ns; - break; - case BPF_FUNC_ktime_get_coarse_ns: - addr = (long)&ktime_get_coarse_ns; - break; - case BPF_FUNC_jiffies64: - addr = (long)&get_jiffies_64; - break; - case BPF_FUNC_test_func: - addr = (long)&bpf_test_func; - break; - default: - err = -EFAULT; - goto out_err; - } - *insn = BPF_EMIT_CALL(addr); - if ((long)__bpf_call_base + insn->imm != addr) - *insn = BPF_JMP_A(0); /* Skip: NOP */ - break; - } - } - - fp = bpf_prog_select_runtime(fp, &err); - if (err) - goto out_err; - - progs->ptrs[which] = fp; - } - - /* The last entry contains a NULL program pointer */ - progs->map.max_entries = ntests + 1; - *pprogs = progs; - return 0; - -out_nomem: - err = -ENOMEM; - -out_err: - if (progs) - destroy_tail_call_tests(progs); - return err; -} - -static __init int test_tail_calls(struct bpf_array *progs) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - struct tail_call_test *test = &tail_call_tests[i]; - struct bpf_prog *fp = progs->ptrs[i]; - int *data = NULL; - int state = 0; - u64 duration; - int ret; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - if (!fp) { - err_cnt++; - continue; - } - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - if (test->flags & FLAG_NEED_STATE) - data = &state; - ret = __run_one(fp, data, MAX_TESTRUNS, &duration); - if (test->flags & FLAG_RESULT_IN_STATE) - ret = state; - if (ret == test->result) { - pr_cont("%lld PASS", duration); - pass_cnt++; - } else { - pr_cont("ret %d != %d FAIL", ret, test->result); - err_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - __func__, pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -static char test_suite[32]; -module_param_string(test_suite, test_suite, sizeof(test_suite), 0); - -static __init int find_test_index(const char *test_name) -{ - int i; - - if (!strcmp(test_suite, "test_bpf")) { - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_tail_calls")) { - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - if (!strcmp(tail_call_tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_skb_segment")) { - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - if (!strcmp(skb_segment_tests[i].descr, test_name)) - return i; - } - } - - return -1; -} - -static __init int prepare_test_range(void) -{ - int valid_range; - - if (!strcmp(test_suite, "test_bpf")) - valid_range = ARRAY_SIZE(tests); - else if (!strcmp(test_suite, "test_tail_calls")) - valid_range = ARRAY_SIZE(tail_call_tests); - else if (!strcmp(test_suite, "test_skb_segment")) - valid_range = ARRAY_SIZE(skb_segment_tests); - else - return 0; - - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= valid_range) { - pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", - test_name, test_suite); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] < 0 || test_range[1] >= valid_range) { - pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - return 0; -} - -static int __init test_bpf_init(void) -{ - struct bpf_array *progs = NULL; - int ret; - - if (strlen(test_suite) && - strcmp(test_suite, "test_bpf") && - strcmp(test_suite, "test_tail_calls") && - strcmp(test_suite, "test_skb_segment")) { - pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); - return -EINVAL; - } - - /* - * if test_suite is not specified, but test_id, test_name or test_range - * is specified, set 'test_bpf' as the default test suite. - */ - if (!strlen(test_suite) && - (test_id != -1 || strlen(test_name) || - (test_range[0] != 0 || test_range[1] != INT_MAX))) { - pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); - strscpy(test_suite, "test_bpf", sizeof(test_suite)); - } - - ret = prepare_test_range(); - if (ret < 0) - return ret; - - if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { - ret = test_bpf(); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { - ret = prepare_tail_call_tests(&progs); - if (ret) - return ret; - ret = test_tail_calls(progs); - destroy_tail_call_tests(progs); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) - return test_skb_segment(); - - return 0; -} - -static void __exit test_bpf_exit(void) -{ -} - -module_init(test_bpf_init); -module_exit(test_bpf_exit); - -MODULE_DESCRIPTION("Testsuite for BPF interpreter and BPF JIT compiler"); -MODULE_LICENSE("GPL"); diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 0f6070a0c8e7..846a58e56203 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -394,9 +394,6 @@ static void probe_kernel_image_config(const char *define_prefix) /* xt_bpf module for passing BPF programs to netfilter */ { "CONFIG_NETFILTER_XT_MATCH_BPF", }, - /* test_bpf module for BPF tests */ - { "CONFIG_TEST_BPF", }, - /* Misc configs useful in BPF C programs */ /* jiffies <-> sec conversion for bpf_jiffies64() helper */ { "CONFIG_HZ", true, } diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 80d8bbbc3084..1e4af62e8312 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -38,8 +38,7 @@ TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ bpftool_btf_dump_unsorted.expected # Order correspond to 'make run_tests' order -TEST_PROGS := test_kmod.sh \ - test_bpftool_build.sh \ +TEST_PROGS := test_bpftool_build.sh \ test_doc_build.sh \ test_xsk.sh \ test_xdp_features.sh diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index d1a19fb125dd..b3d4cf5987a9 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -124,7 +124,6 @@ CONFIG_SAMPLE_LIVEPATCH=m CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_SYN_COOKIES=y -CONFIG_TEST_BPF=m CONFIG_TMPFS=y CONFIG_TMPFS_XATTR=y CONFIG_UDMABUF=y diff --git a/tools/testing/selftests/bpf/test_kmod.sh b/tools/testing/selftests/bpf/test_kmod.sh deleted file mode 100755 index 50dca53ac536..000000000000 --- a/tools/testing/selftests/bpf/test_kmod.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# Usage: -# ./test_kmod.sh [module_param]... -# Ex.: ./test_kmod.sh test_range=1,3 -# All the parameters are passed to the kernel module. - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 - -msg="skip all tests:" -if [ "$(id -u)" != "0" ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -if [ "$building_out_of_srctree" ]; then - # We are in linux-build/kselftest/bpf - OUTPUT=../../ -else - # We are in linux/tools/testing/selftests/bpf - OUTPUT=../../../../ -fi - -test_run() -{ - sysctl -w net.core.bpf_jit_enable=$1 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$2 2>&1 > /dev/null - - echo "[ JIT enabled:$1 hardened:$2 ]" - shift 2 - dmesg -C - if [ -f ${OUTPUT}/lib/test_bpf.ko ]; then - insmod ${OUTPUT}/lib/test_bpf.ko "$@" 2> /dev/null - if [ $? -ne 0 ]; then - rc=1 - fi - else - # Use modprobe dry run to check for missing test_bpf module - if ! /sbin/modprobe -q -n test_bpf "$@"; then - echo "test_bpf: [SKIP]" - elif /sbin/modprobe -q test_bpf "$@"; then - echo "test_bpf: ok" - else - echo "test_bpf: [FAIL]" - rc=1 - fi - fi - rmmod test_bpf 2> /dev/null - dmesg | grep FAIL -} - -test_save() -{ - JE=`sysctl -n net.core.bpf_jit_enable` - JH=`sysctl -n net.core.bpf_jit_harden` -} - -test_restore() -{ - sysctl -w net.core.bpf_jit_enable=$JE 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$JH 2>&1 > /dev/null -} - -rc=0 -test_save -test_run 0 0 "$@" -test_run 1 0 "$@" -test_run 1 1 "$@" -test_run 1 2 "$@" -test_restore -exit $rc diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 590b33e16d9f..0ffce8efc6d0 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -104,7 +104,6 @@ TEST_PROGS := \ stress_reuseport_listen.sh \ tcp_ecmp_failover.sh \ tcp_fastopen_backup_key.sh \ - test_bpf.sh \ test_bridge_backup_port.sh \ test_bridge_neigh_suppress.sh \ test_ingress_egress_chaining.sh \ diff --git a/tools/testing/selftests/net/config b/tools/testing/selftests/net/config index 737e7e6327b3..6223b00516fe 100644 --- a/tools/testing/selftests/net/config +++ b/tools/testing/selftests/net/config @@ -127,7 +127,6 @@ CONFIG_TAP=m CONFIG_TCP_CONG_DCTCP=y CONFIG_TCP_MD5SIG=y CONFIG_TEST_BLACKHOLE_DEV=m -CONFIG_TEST_BPF=m CONFIG_TLS=m CONFIG_TRACEPOINTS=y CONFIG_TUN=y diff --git a/tools/testing/selftests/net/test_bpf.sh b/tools/testing/selftests/net/test_bpf.sh deleted file mode 100755 index 65677909c574..000000000000 --- a/tools/testing/selftests/net/test_bpf.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# Runs bpf test using test_bpf kernel module - -if /sbin/modprobe -q test_bpf ; then - /sbin/modprobe -q -r test_bpf; - echo "test_bpf: ok"; -else - echo "test_bpf: [FAIL]"; - exit 1; -fi From f5bffe9f098fbf1b60e0242c7b96e06d1e4256d5 Mon Sep 17 00:00:00 2001 From: Marco Vedovati Date: Wed, 14 Oct 2026 21:45:16 +0200 Subject: [PATCH 505/660] tools/include: Add Linux-syscall-note to uapi/linux/types.h tools/include/uapi/linux/types.h is the only header under tools/include/uapi/linux/ whose SPDX identifier lacks the Linux-syscall-note exception. The file was originally added as tools/testing/selftests/bpf/include/uapi/linux/types.h by commit 0a5539f66133 ("bpf: Provide a linux/types.h override for bpf selftests.") without any license header. Then commit 579f1d926c66 ("selftests/bpf: fix broken build due to types.h") added type definitions copied from include/uapi/linux/types.h, which is licensed under GPL-2.0 WITH Linux-syscall-note. Commit b24413180f56 ("License cleanup: add SPDX GPL-2.0 license identifier to files with no license") tagged it GPL-2.0, along with other headers under tools/include/uapi/. Those other headers were switched to GPL-2.0 WITH Linux-syscall-note shortly after by commit fb7df12d645c ("tools/headers: Synchronize kernel ABI headers"), when they were synchronized with their kernel counterparts. This file is only a partial copy of include/uapi/linux/types.h, not kept in sync with it, so it kept the GPL-2.0 identifier, also after commit bcd66b10b5e9 ("tools/bpf: Move linux/types.h for selftests and bpftool") moved it to its current location. Use the same identifier as include/uapi/linux/types.h. Signed-off-by: Marco Vedovati Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261014194516.3030362-1-marco.vedovati@crowdstrike.com --- tools/include/uapi/linux/types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/include/uapi/linux/types.h b/tools/include/uapi/linux/types.h index 85aa327245c6..6acf2059085a 100644 --- a/tools/include/uapi/linux/types.h +++ b/tools/include/uapi/linux/types.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_LINUX_TYPES_H #define _UAPI_LINUX_TYPES_H From 76c1913f311f32971dc2c3e106e285bd63c1804d Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 8 Oct 2026 22:02:39 +0800 Subject: [PATCH 506/660] selftests/bpf: Use its own netns in flow_dissector_skb_less_direct_attach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_flow_dissector_skb_less_direct_attach() and test_flow_dissector_skb_less_indirect_attach() each create a netns to run in, and test_progs may run the two at the same time. Both name their netns flow_dissector_skb_less_indirect_attach_ns. When the two tests overlap, one of them fails to create or open the netns, or loses it to the cleanup of the other one. Name the netns of the direct attach test after that test. Fixes: 2b044dd186f0 ("selftests/bpf: re-split main function into dedicated tests") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Reviewed-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/bpf/20261008-selftests-bpf-flow-dissector-netns-v1-1-5d9889aaad84@gmail.com --- tools/testing/selftests/bpf/prog_tests/flow_dissector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c index fb4892681464..f6fefb14cda3 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c @@ -706,7 +706,7 @@ void test_flow_dissector_skb_less_direct_attach(void) struct bpf_flow *skel; struct netns_obj *ns; - ns = netns_new("flow_dissector_skb_less_indirect_attach_ns", true); + ns = netns_new("flow_dissector_skb_less_direct_attach_ns", true); if (!ASSERT_OK_PTR(ns, "create and open netns")) return; From 66fd5eb9da12ef4ed089ed1974160d07ff034a19 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 8 Oct 2026 22:03:25 +0800 Subject: [PATCH 507/660] selftests/bpf: Report a failure to register the session key in test_progs test_progs adds the key that verifies signed programs to the session keyring before it runs any test, and exits when that fails. It exits with 255 without printing anything. pam_keyinit revokes the session keyring on logout, so a test_progs started in the background from an ssh session that has been closed since leaves nothing but an empty log behind, with no hint of what went wrong. Print the error before exiting. Fixes: b720903e2b14 ("selftests/bpf: Enable signature verification for some lskel tests") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261008-selftests-bpf-test-progs-session-key-v1-1-ec48e793639b@gmail.com --- tools/testing/selftests/bpf/test_progs.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 46eb201b96a3..168d84cf4eb7 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -2065,8 +2065,11 @@ int main(int argc, char **argv) libbpf_set_print(libbpf_print_fn); err = register_session_key((const char *)test_progs_verification_cert, test_progs_verification_cert_len); - if (err < 0) + if (err < 0) { + fprintf(stderr, "Failed to add libbpf_session_key to the session keyring: %s\n", + strerror(errno)); return err; + } traffic_monitor_set_print(traffic_monitor_print_fn); From a5e144985d6855c84b56ea5443fd6333490ce8ec Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 8 Oct 2026 22:03:47 +0800 Subject: [PATCH 508/660] selftests/bpf: Expect at least one tp_recursion miss in missed missed_tp_recursion attaches its tp programs to bpf_trace_printk and expects each of them to miss exactly one run, the one that its kprobe program on bpf_fentry_test1 triggers by calling bpf_printk(). The test is not serial, so tests running in parallel can add misses to the tp programs while they are attached: - The kprobe on bpf_fentry_test1 is global. Any other test that does bpf_prog_test_run() on a fentry/fexit program calls bpf_fentry_test1() and thereby fires the kprobe program, which calls bpf_printk() from kprobe context and makes the tp programs fail the bpf_prog_active check once more. - Any other program that runs under the bpf_prog_active check, such as a kprobe program or the perf event tracepoint program in legacy_printk, and calls bpf_printk() hits bpf_trace_printk the same way. Either way, test2_recursion_misses comes out as 2 instead of 1. Expect at least one miss, as missed_kprobe_recursion does since commit 08a7078feacf ("selftests/bpf: Improve missed_kprobe_recursion test robustness"). Fixes: 85981e0f9e9f ("selftests/bpf: Add test for recursion counts of perf event link tracepoint") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261008-selftests-bpf-missed-tp-recursion-v1-1-0dd60471d655@gmail.com --- tools/testing/selftests/bpf/prog_tests/missed.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/missed.c b/tools/testing/selftests/bpf/prog_tests/missed.c index ed8857ae914a..0b8f1aa5a3e7 100644 --- a/tools/testing/selftests/bpf/prog_tests/missed.c +++ b/tools/testing/selftests/bpf/prog_tests/missed.c @@ -120,9 +120,9 @@ static void test_missed_tp_recursion(void) ASSERT_EQ(topts.retval, 0, "test_run"); ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test1)), 0, "test1_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); cleanup: missed_tp_recursion__destroy(skel); From 7f34009230b796a06198da2c7204fd4efab9a2a1 Mon Sep 17 00:00:00 2001 From: Michal Luczaj Date: Thu, 8 Oct 2026 19:12:15 +0200 Subject: [PATCH 509/660] selftests/bpf: Clean up sa_family_t-stringifier Following commit f3de1cf621f7 ("selftests/bpf: sockmap_listen cleanup: Drop af_unix redir tests") and commit 9266e49d608c ("selftests/bpf: sockmap_listen cleanup: Drop af_vsock redir tests"), sockmap_listen.c: family_str() becomes functionally identical to select_reuseport.c: family_str(). Move the logic to socket_helpers.h. Suggested-by: Kuniyuki Iwashima Signed-off-by: Michal Luczaj Signed-off-by: Daniel Borkmann Reviewed-by: Kuniyuki Iwashima Link: https://msgid.link/CAAVpQUDULzZh58oKkRaYqgM9bRToGb7wHrzT+cZNpRmAtM12Ow@mail.gmail.com Link: https://lore.kernel.org/bpf/20261008-selftests-sockmap_listen-cleanup-v1-1-f6d1a4ff540d@rbox.co --- .../bpf/prog_tests/select_reuseport.c | 16 +++----------- .../selftests/bpf/prog_tests/socket_helpers.h | 12 ++++++++++ .../selftests/bpf/prog_tests/sockmap_listen.c | 22 +++---------------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c index 3dbcc091f16c..3b4b233e21d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include "bpf_util.h" #include "test_progs.h" +#include "socket_helpers.h" #include "test_select_reuseport_common.h" #define MAX_TEST_NAME 80 @@ -740,18 +742,6 @@ static const char *maptype_str(enum bpf_map_type type) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - default: - return "unknown"; - } -} - static const char *sotype_str(int sotype) { switch (sotype) { @@ -794,7 +784,7 @@ static void test_config(int sotype, sa_family_t family, bool inany) snprintf(s, sizeof(s), "%s %s/%s %s %s", maptype_str(inner_map_type), - family_str(family), sotype_str(sotype), + inet_family_str(family), sotype_str(sotype), inany ? "INANY" : "LOOPBACK", t->name); if (!test__start_subtest(s)) diff --git a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h index 0d59503a0c73..4a570998612d 100644 --- a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h +++ b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h @@ -470,4 +470,16 @@ static inline const char *socket_kind_to_str(int sock_fd) return "???"; } +static inline const char *inet_family_str(sa_family_t family) +{ + switch (family) { + case AF_INET: + return "IPv4"; + case AF_INET6: + return "IPv6"; + default: + return "unknown"; + } +} + #endif // __SOCKET_HELPERS__ diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 1c96a3cf4b97..235e1d56f044 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -1232,22 +1232,6 @@ static void test_ops_cleanup(const struct bpf_map *map) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - case AF_UNIX: - return "Unix"; - case AF_VSOCK: - return "VSOCK"; - default: - return "unknown"; - } -} - static const char *map_type_str(const struct bpf_map *map) { int type; @@ -1315,7 +1299,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, char s[MAX_TEST_NAME]; int map_fd; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype); map_fd = bpf_map__fd(map); @@ -1355,7 +1339,7 @@ static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map, const struct redir_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { @@ -1387,7 +1371,7 @@ static void test_reuseport(struct test_sockmap_listen *skel, const struct reuseport_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype); From 33cbc5c2cadfe3b43f024e89880dd8fe2efb3a2e Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 09:48:44 +0800 Subject: [PATCH 510/660] selftests/bpf: Create the vxlan devices of test_tunnel with their MAC The vxlan tests of test_tunnel create a tunnel device in the initial netns with "ip link add", and then give it the MAC address that the static neighbor entry at the other end points to with "ip link set". systemd-udevd assigns new devices of the initial netns a MAC address of its own, following its MACAddressPolicy, when it handles their uevent. When it gets to that after the test set the address, the replies sent to the configured MAC are dropped as meant for another host, and the ping of vxlan_tunnel fails after 10 seconds. Pass the address to "ip link add". udevd leaves alone the address of a device that was created with one. Fixes: 1ee7efd40abf ("selftests/bpf: Move vxlan tunnel testcases to test_progs") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261009-selftests-bpf-tunnel-vxlan-mac-v1-1-624dd769c334@gmail.com --- .../testing/selftests/bpf/prog_tests/test_tunnel.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c index eb9309931272..7b35bf949805 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c @@ -161,9 +161,9 @@ static int add_vxlan_tunnel(void) IP4_ADDR2_VETH1, MAC_VETH1); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external gbp dstport 4789", - VXLAN_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link add dev %s address %s type vxlan external gbp dstport 4789", + VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", VXLAN_TUNL_DEV1); SYS(fail, "ip addr add dev %s %s/24", VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); SYS(fail, "ip neigh add %s lladdr %s dev %s", IP4_ADDR_TUNL_DEV0, MAC_TUNL_DEV0, VXLAN_TUNL_DEV1); @@ -198,11 +198,10 @@ static int add_ip6vxlan_tunnel(void) IP6VXLAN_TUNL_DEV0, MAC_TUNL_DEV0); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external dstport 4789", - IP6VXLAN_TUNL_DEV1); - SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", + SYS(fail, "ip link add dev %s address %s type vxlan external dstport 4789", IP6VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", IP6VXLAN_TUNL_DEV1); return 0; fail: From d3c9ecb8765d4befbc46ee325c8d76efd523eece Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Tue, 13 Oct 2026 11:01:14 +0800 Subject: [PATCH 511/660] selftests/bpf: Fix map fd leak in no-BTF wq test bpf_map__reuse_fd() duplicates the supplied file descriptor instead of taking ownership of it. test_failure_map_no_btf() closes the original map_fd only when reuse fails, leaking it when reuse succeeds, including when the test passes. Destroying the skeleton closes its own descriptor but leaves the original descriptor and its map reference alive. Close map_fd immediately after bpf_map__reuse_fd(), before checking its return value, to release the original descriptor on both success and failure. The skeleton retains its own descriptor for the subsequent program load. Fixes: bca2b74ea9a8 ("selftests/bpf: Add more bpf_wq tests") Reported-by: Jackie Liu Signed-off-by: Yun Lu Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013030114.292157-1-luyun_611@163.com --- tools/testing/selftests/bpf/prog_tests/wq.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 84831eecc935..97926e35aaf2 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -64,10 +64,9 @@ static void test_failure_map_no_btf(void) goto out; err = bpf_map__reuse_fd(skel->maps.array, map_fd); - if (!ASSERT_OK(err, "map reuse fd")) { - close(map_fd); + close(map_fd); + if (!ASSERT_OK(err, "map reuse fd")) goto out; - } insns = bpf_program__insns(skel->progs.test_map_no_btf); if (!ASSERT_OK_PTR(insns, "insns ptr")) From 653b8f14fdda385bd7c5b786f8dfdcc3e5599e27 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 09:49:42 +0800 Subject: [PATCH 512/660] selftests/bpf: Accept unnamed dma-bufs in the dmabuf_iter default test The dmabuf_collector iterator prints the inode, size, name and exporter of every dma-buf in the system on separate lines, with an empty name for the buffers that have none. The default_iter subtest parses that output and looks for the two buffers it created. It requires a name for every buffer, and sscanf() finds none on the empty line of an unnamed one. On a system that has such a buffer, like one exported by udmabuf without a name, the subtest fails before it gets to its own buffers. Take an empty name for a buffer without one. Fixes: ae5d2c59ecd7 ("selftests/bpf: Add test for dmabuf_iter") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Reviewed-by: T.J. Mercier Link: https://lore.kernel.org/bpf/20261009-selftests-bpf-dmabuf-iter-unnamed-v1-1-962ee94f7a5e@gmail.com --- tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c index fb2cea710db3..1a5dc1adae73 100644 --- a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c @@ -188,8 +188,9 @@ static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel) ASSERT_EQ(sscanf(line, "%ld", &bufinfo.size), 1, "read size"); } else if (f % FIELD_COUNT == NAME) { - ASSERT_EQ(sscanf(line, "%s", bufinfo.name), 1, - "read name"); + /* Keep an empty name for a buffer without one */ + if (sscanf(line, "%s", bufinfo.name) != 1) + bufinfo.name[0] = '\0'; } else if (f % FIELD_COUNT == EXPORTER) { ASSERT_EQ(sscanf(line, "%31s", bufinfo.exporter), 1, "read exporter"); From 26c9dc906692104f65ae33721ff5ac2a0ca0c645 Mon Sep 17 00:00:00 2001 From: "T.J. Mercier" Date: Thu, 8 Oct 2026 03:41:05 -0700 Subject: [PATCH 513/660] bpf: htab: Split htab_elem_lru and htab_elem_pcpu off of htab_elem The htab_elem struct is used as the per-element type for all BPF hash map types and includes bpf_lru_node in a union with a ptr_to_pptr pointer. For standard (non-LRU, non-PCPU) hash maps, the 24 byte union allocated for every element is entirely unused. For non-preallocated PCPU maps, ptr_to_pptr only requires 8 bytes, leaving 16 bytes of unused overhead in the union. For preallocated PCPU maps ptr_to_pptr is unused since elements are freed to the PCPU freelist. Eliminate this per-element memory overhead by splitting htab_elem into dedicated structures for each map type: - struct htab_elem: Minimal structure for standard hash maps and preallocated PCPU maps (saves 24 bytes per element). - struct htab_elem_pcpu: Structure for non-preallocated PCPU maps containing ptr_to_pptr (saves 16 bytes per element). - struct htab_elem_lru: Retains struct bpf_lru_node for LRU maps. Place lru_node and ptr_to_pptr before struct htab_elem in htab_elem_lru and htab_elem_pcpu respectively, and track the offset of htab_elem from the start of the element allocation in htab->elem_offset. This keeps key and value at constant compile-time offsets from struct htab_elem across all hash map types, avoiding dynamic key offset calculations on lookups. Use sizeof(struct htab_elem_lru) for the element size rollover check in htab_map_alloc_check() since it is now the largest element variant. Signed-off-by: T.J. Mercier Signed-off-by: Alexei Starovoitov Acked-by: Mykyta Yatsenko Link: https://patch.msgid.link/20261008104108.993791-2-tjmercier@google.com --- kernel/bpf/hashtab.c | 140 ++++++++++++------ .../selftests/bpf/progs/map_ptr_kern.c | 2 +- 2 files changed, 92 insertions(+), 50 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 426254f979a8..810db9c43653 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -102,6 +102,7 @@ struct bpf_htab { bool use_percpu_counter; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ + u32 elem_offset;/* offset of htab_elem in bytes */ u32 hashrnd; }; @@ -117,15 +118,25 @@ struct htab_elem { }; }; }; - union { - /* pointer to per-cpu pointer */ - void *ptr_to_pptr; - struct bpf_lru_node lru_node; - }; - u32 hash; + u32 hash __aligned(8); char key[] __aligned(8); }; +struct htab_elem_lru { + struct bpf_lru_node lru_node; + struct htab_elem elem; +}; + +/* + * Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need + * ptr_to_pptr, and use htab_elem. + */ +struct htab_elem_pcpu { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem elem; +}; + struct htab_btf_record { struct btf_record *record; struct btf *btf; @@ -184,6 +195,26 @@ static inline bool is_fd_htab(const struct bpf_htab *htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } +static void *htab_elem_container(const struct bpf_htab *htab, struct htab_elem *l) +{ + return (void *)l - htab->elem_offset; +} + +static void *htab_elem_get_ptr_to_pptr(struct htab_elem *l) +{ + return container_of(l, struct htab_elem_pcpu, elem)->ptr_to_pptr; +} + +static void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr) +{ + container_of(l, struct htab_elem_pcpu, elem)->ptr_to_pptr = ptr; +} + +static struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l) +{ + return &container_of(l, struct htab_elem_lru, elem)->lru_node; +} + static inline void *htab_elem_value(struct htab_elem *l, u32 key_size) { return l->key + round_up(key_size, 8); @@ -207,7 +238,7 @@ static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) { - return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); + return htab->elems + i * (u64)htab->elem_size + htab->elem_offset; } /* Both percpu and fd htab support in-place update, so no need for @@ -301,16 +332,16 @@ free_elems: * bucket_lock followed by lru_lock is not allowed. In such cases, * bucket_lock needs to be released first before acquiring lru_lock. */ -static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, - u32 hash) +static struct htab_elem_lru *prealloc_lru_pop(struct bpf_htab *htab, void *key, + u32 hash) { struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); - struct htab_elem *l; + struct htab_elem_lru *l; if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = container_of(node, struct htab_elem_lru, lru_node); + memcpy(l->elem.key, key, htab->map.key_size); return l; } @@ -350,8 +381,8 @@ skip_percpu_elems: if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, elem.hash) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -362,11 +393,12 @@ skip_percpu_elems: if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, - htab->elems + offsetof(struct htab_elem, fnode), + htab->elems + htab->elem_offset + + offsetof(struct htab_elem, fnode), htab->elem_size, num_entries); return 0; @@ -454,7 +486,7 @@ static int htab_map_alloc_check(union bpf_attr *attr) return -EINVAL; if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE - - sizeof(struct htab_elem)) + sizeof(struct htab_elem_lru)) /* if key_size + value_size is bigger, the user space won't be * able to access the elements via bpf syscall. This check * also makes sure that the elem_size doesn't overflow and it's @@ -601,7 +633,13 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + + if (htab_is_lru(htab)) + htab->elem_offset = offsetof(struct htab_elem_lru, elem); + else if (percpu && !prealloc) + htab->elem_offset = offsetof(struct htab_elem_pcpu, elem); + + htab->elem_size = htab->elem_offset + + sizeof(struct htab_elem) + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void *); @@ -844,7 +882,7 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return htab_elem_value(l, map->key_size); } @@ -867,19 +905,17 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; + const s16 ref_off = (int)offsetof(struct htab_elem_lru, lru_node) + + (int)offsetof(struct bpf_lru_node, ref) - + (int)offsetof(struct htab_elem_lru, elem); BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); - *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref)); + *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, ref_off); *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); - *insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref), - 1); + *insn++ = BPF_ST_MEM(BPF_B, ret, ref_off, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + round_up(map->key_size, 8)); @@ -911,15 +947,16 @@ static void check_and_cancel_fields(struct bpf_htab *htab, static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) { struct bpf_htab *htab = arg; - struct htab_elem *l = NULL, *tgt_l; + struct htab_elem_lru *tgt_l; + struct htab_elem *l = NULL; struct hlist_nulls_head *head; struct hlist_nulls_node *n; unsigned long flags; struct bucket *b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); - b = __select_bucket(htab, tgt_l->hash); + tgt_l = container_of(node, struct htab_elem_lru, lru_node); + b = __select_bucket(htab, tgt_l->elem.hash); head = &b->head; ret = htab_lock_bucket(b, &flags); @@ -927,7 +964,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) return false; hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l == tgt_l) { + if (l == &tgt_l->elem) { hlist_nulls_del_rcu(&l->hash_node); bpf_map_dec_elem_count(&htab->map); break; @@ -935,9 +972,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); - if (l == tgt_l) + if (l == &tgt_l->elem) check_and_cancel_fields(htab, l); - return l == tgt_l; + return l == &tgt_l->elem; } /* Called from syscall */ @@ -1004,8 +1041,8 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); - bpf_mem_cache_free(&htab->ma, l); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l)); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l)); } static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) @@ -1153,6 +1190,8 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, bpf_map_inc_elem_count(&htab->map); } } else { + void *container; + if (is_map_full(htab)) if (!old_elem) /* when map is full and update() is replacing @@ -1162,11 +1201,12 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, */ return ERR_PTR(-E2BIG); inc_elem_count(htab); - l_new = bpf_mem_cache_alloc(&htab->ma); - if (!l_new) { + container = bpf_mem_cache_alloc(&htab->ma); + if (!container) { l_new = ERR_PTR(-ENOMEM); goto dec_count; } + l_new = container + htab->elem_offset; } memcpy(l_new->key, key, key_size); @@ -1178,11 +1218,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); if (!ptr) { - bpf_mem_cache_free(&htab->ma, l_new); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l_new)); l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(l_new, ptr); pptr = *(void __percpu **)ptr; } @@ -1327,14 +1367,15 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new, *l_old = NULL; + struct htab_elem *l_old = NULL; + struct htab_elem_lru *l_new; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1362,7 +1403,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(&l_new->elem, map->key_size), value); ret = htab_lock_bucket(b, &flags); if (ret) @@ -1377,7 +1418,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value /* add new element to the head of the list, so that * concurrent search will find it before old elem */ - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); if (l_old) { bpf_lru_node_set_ref(&l_new->lru_node); hlist_nulls_del_rcu(&l_old->hash_node); @@ -1389,7 +1430,7 @@ err: err_lock_bucket: if (ret) - htab_lru_push_free(htab, l_new); + htab_lru_push_free(htab, &l_new->elem); else if (l_old) htab_lru_push_free(htab, l_old); @@ -1473,7 +1514,8 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, bool onallcpus) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new = NULL, *l_old; + struct htab_elem_lru *l_new = NULL; + struct htab_elem *l_old; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1515,15 +1557,15 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); /* per-cpu hash map can update value in-place */ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(&l_new->elem, key_size), value, onallcpus, map_flags); - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); l_new = NULL; } ret = 0; @@ -2520,7 +2562,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) struct htab_elem *l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); } @@ -2536,7 +2578,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); } diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index 373c8d17ea55..f71be4fc8dd7 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 64); + VERIFY(hash->elem_size == 40); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0); From 0d0421c96f67a4dceb6cfd02f440a71849a4bab4 Mon Sep 17 00:00:00 2001 From: "T.J. Mercier" Date: Thu, 8 Oct 2026 03:41:06 -0700 Subject: [PATCH 514/660] bpf: htab: Reduce elem_size by 8 bytes for small key sizes For standard and PCPU (non-LRU) hash maps with small key sizes (less than or equal to the word size), comparing keys requires only a single instruction. Storing a cached 32-bit hash value to shortcut full key comparisons provides no performance advantage for small keys, and consumes memory for every element. This memory can be saved by removing hash from struct htab_elem and placing it directly before struct htab_elem only for the new htab_elem_hashed type which is used only when keys are larger than the word size or for LRU maps. This reduces elem_size by 8 bytes for small keys while keeping key and hash at constant compile-time offsets from struct htab_elem across all map types and key sizes. All element variants requiring hashes get an anonymous htab_elem_hashed embedding, ensuring that the -8 byte hash offset is guaranteed for all element types by composition. Elements can be recycled without a RCU grace period. Before this commit, alloc_htab_elem() overwrote the key before initializing the value/pptr and wrote the hash last, so during value assignment on a recycled element, the new key was paired with the old hash preventing lockless readers from matching either the old key or the new key except when there was also a hash collision. When hashes are omitted for small keys, only the key field guards lookups. Writing the new key before value/pptr initialization would allow a concurrent lookup of the new key to match an uninitialized value or dereference a stale or freed pptr. So move the new key assignment to the end of alloc_htab_elem() where the hash assignment was done. Like the hash assignment it replaces, it does not enforce value store ordering before key stores on weakly ordered architectures for lockless readers that were already traversing l_new via a stale pointer before hlist_nulls_add_head_rcu(). Before this commit htab_mem_dtor() was invoked with the start of the allocation, which is no longer always the address of struct htab_elem now that elem_offset can be nonzero for non-preallocated, non-per-CPU maps. So the key_size field of struct htab_btf_record has been replaced with a value_offset computed at map creation and passed through bpf_ma_set_dtor(). That reduces the destructor to a call to bpf_obj_free_fields() at a fixed offset, which is exactly what rhtab_mem_dtor() did, so rhtab_mem_dtor() has been removed and replaced with the new implementation of htab_mem_dtor() for BPF_MAP_TYPE_RHASH. Together with the previous patch, this reduces the minimum standard and preallocated hash map element size from 64 bytes down to 32 bytes, and non-preallocated per-CPU element size from 64 bytes down to 40 bytes. Signed-off-by: T.J. Mercier Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261008104108.993791-3-tjmercier@google.com --- kernel/bpf/hashtab.c | 229 +++++++++++++----- .../selftests/bpf/progs/map_ptr_kern.c | 2 +- 2 files changed, 166 insertions(+), 65 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 810db9c43653..9d4791ee1741 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -100,6 +100,7 @@ struct bpf_htab { struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; + bool has_hash; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ u32 elem_offset;/* offset of htab_elem in bytes */ @@ -118,18 +119,22 @@ struct htab_elem { }; }; }; - u32 hash __aligned(8); char key[] __aligned(8); }; +struct htab_elem_hashed { + u32 hash __aligned(8); + struct htab_elem elem; +}; + struct htab_elem_lru { struct bpf_lru_node lru_node; - struct htab_elem elem; + struct htab_elem_hashed; }; /* - * Only for non-preallocated PCPU maps. Preallocated PCPU maps don't need - * ptr_to_pptr, and use htab_elem. + * Only for non-preallocated PCPU maps with small keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem. */ struct htab_elem_pcpu { /* pointer to per-cpu pointer */ @@ -137,10 +142,23 @@ struct htab_elem_pcpu { struct htab_elem elem; }; +/* + * Only for non-preallocated PCPU maps with large keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem_hashed. + */ +struct htab_elem_pcpu_hashed { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem_hashed; +}; + +static_assert(offsetof(struct htab_elem_pcpu_hashed, ptr_to_pptr) == + offsetof(struct htab_elem_pcpu, ptr_to_pptr)); + struct htab_btf_record { struct btf_record *record; struct btf *btf; - u32 key_size; + u32 value_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab *htab) @@ -195,19 +213,64 @@ static inline bool is_fd_htab(const struct bpf_htab *htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } +/* + * u32 hash reads are always atomic. If the hash is elided, key comparisons + * must also be atomic to avoid false positive matches from torn key + * reads/writes on recycled elements. That is only possible when the + * key fits within a word. + */ +static __always_inline bool htab_key_size_needs_hash(u32 key_size) +{ + return key_size > sizeof(unsigned long); +} + +/* + * When hash is omitted, key reads and writes must be atomic to support lockless + * RCU readers. Zero-extend keys to the word size to support all sub-word key + * sizes, regardless of the size of the caller's key allocation. + */ +static __always_inline unsigned long htab_zero_extend_key(const void *key, + u32 key_size) +{ + unsigned long k = 0; + + memcpy(&k, key, min_t(u32, key_size, sizeof(k))); + return k; +} + +static bool htab_has_hash(const struct bpf_htab *htab) +{ + return htab->has_hash; +} + +static u32 htab_elem_hash(struct htab_elem *l) +{ + return container_of(l, struct htab_elem_hashed, elem)->hash; +} + +static void htab_elem_set_hash(struct htab_elem *l, u32 hash) +{ + container_of(l, struct htab_elem_hashed, elem)->hash = hash; +} + static void *htab_elem_container(const struct bpf_htab *htab, struct htab_elem *l) { return (void *)l - htab->elem_offset; } -static void *htab_elem_get_ptr_to_pptr(struct htab_elem *l) +static void *htab_elem_get_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l) { - return container_of(l, struct htab_elem_pcpu, elem)->ptr_to_pptr; + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + return pcpu_elem->ptr_to_pptr; } -static void htab_elem_set_ptr_to_pptr(struct htab_elem *l, void *ptr) +static void htab_elem_set_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l, + void *ptr) { - container_of(l, struct htab_elem_pcpu, elem)->ptr_to_pptr = ptr; + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + pcpu_elem->ptr_to_pptr = ptr; } static struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l) @@ -381,7 +444,7 @@ skip_percpu_elems: if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem_lru, elem.hash) - + offsetof(struct htab_elem_lru, hash) - offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); @@ -503,14 +566,11 @@ static int htab_map_alloc_check(union bpf_attr *attr) static void htab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; - struct htab_elem *elem = obj; - void *map_value; if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); - bpf_obj_free_fields(hrec->record, map_value); + bpf_obj_free_fields(hrec->record, obj + hrec->value_offset); } static void htab_pcpu_mem_dtor(void *obj, void *ctx) @@ -540,7 +600,7 @@ static void htab_dtor_ctx_free(void *ctx) } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, - void (*dtor)(void *, void *)) + void (*dtor)(void *, void *), u32 value_offset) { struct htab_btf_record *hrec; int err; @@ -552,7 +612,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, hrec = kzalloc_obj(*hrec); if (!hrec) return -ENOMEM; - hrec->key_size = map->key_size; + hrec->value_offset = value_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); @@ -587,9 +647,12 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, * populated in htab_map_alloc(), so it will always appear as NULL. */ if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, 0); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, + htab->elem_offset + + offsetof(struct htab_elem, key) + + round_up(map->key_size, 8)); } static struct bpf_map *htab_map_alloc(union bpf_attr *attr) @@ -612,6 +675,10 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) bpf_map_init_from_attr(&htab->map, attr); + /* Avoid hash memory use and comparisons where unnecessary. */ + htab->has_hash = htab_is_lru(htab) || + htab_key_size_needs_hash(htab->map.key_size); + if (percpu_lru) { /* ensure each CPU's lru list has >=1 elements. * since we are at it, make each lru list has the same @@ -636,7 +703,11 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) if (htab_is_lru(htab)) htab->elem_offset = offsetof(struct htab_elem_lru, elem); else if (percpu && !prealloc) - htab->elem_offset = offsetof(struct htab_elem_pcpu, elem); + htab->elem_offset = htab_has_hash(htab) ? + offsetof(struct htab_elem_pcpu_hashed, elem) : + offsetof(struct htab_elem_pcpu, elem); + else if (htab_has_hash(htab)) + htab->elem_offset = offsetof(struct htab_elem_hashed, elem); htab->elem_size = htab->elem_offset + sizeof(struct htab_elem) + @@ -754,35 +825,57 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 return &__select_bucket(htab, hash)->head; } -/* this lookup function can only be called with bucket lock taken */ -static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, - void *key, u32 key_size) +static __always_inline struct hlist_nulls_node * +__lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) { struct hlist_nulls_node *n; struct htab_elem *l; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (elem_has_hash) { + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (htab_elem_hash(l) == hash && + !memcmp(&l->key, key, key_size)) + break; + } else { + unsigned long k = htab_zero_extend_key(key, key_size); - return NULL; + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (READ_ONCE(*(unsigned long *)l->key) == k) + break; + } + + return n; +} + +/* this lookup function can only be called with bucket lock taken */ +static __always_inline struct htab_elem * +lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) +{ + struct hlist_nulls_node *n; + + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (is_a_nulls(n)) + return NULL; + + return hlist_nulls_entry(n, struct htab_elem, hash_node); } /* can be called without bucket lock. it will repeat the loop in * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static __always_inline struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem * +lookup_nulls_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash, u32 n_buckets) { struct hlist_nulls_node *n; - struct htab_elem *l; again: - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (!is_a_nulls(n)) + return hlist_nulls_entry(n, struct htab_elem, hash_node); if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) goto again; @@ -790,7 +883,8 @@ again: return NULL; } -static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 key_size) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, + u32 key_size, bool elem_has_hash) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; @@ -803,7 +897,8 @@ static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 k head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, elem_has_hash, + htab->n_buckets); return l; } @@ -817,17 +912,25 @@ static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 k */ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) { - return __htab_lookup(map, key, map->key_size); + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + + return __htab_lookup(map, key, map->key_size, htab_has_hash(htab)); } +/* + * Specialized lookups for htab_map_gen_lookup(). These are only used for + * BPF_MAP_TYPE_HASH (never LRU), so elem_has_hash depends only on key size. + */ static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) { - return __htab_lookup(map, key, sizeof(u32)); + return __htab_lookup(map, key, sizeof(u32), + htab_key_size_needs_hash(sizeof(u32))); } static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) { - return __htab_lookup(map, key, sizeof(u64)); + return __htab_lookup(map, key, sizeof(u64), + htab_key_size_needs_hash(sizeof(u64))); } static void *htab_map_lookup_elem(struct bpf_map *map, void *key) @@ -956,7 +1059,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) int ret; tgt_l = container_of(node, struct htab_elem_lru, lru_node); - b = __select_bucket(htab, tgt_l->elem.hash); + b = __select_bucket(htab, tgt_l->hash); head = &b->head; ret = htab_lock_bucket(b, &flags); @@ -998,7 +1101,8 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) head = select_bucket(htab, hash); /* lookup the key */ - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); if (!l) goto find_first_elem; @@ -1041,7 +1145,7 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(l)); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(htab, l)); bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l)); } @@ -1209,7 +1313,9 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, l_new = container + htab->elem_offset; } - memcpy(l_new->key, key, key_size); + if (htab_has_hash(htab)) + memcpy(l_new->key, key, key_size); + if (percpu) { if (prealloc) { pptr = htab_elem_get_ptr(l_new, key_size); @@ -1222,7 +1328,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, l_new = ERR_PTR(-ENOMEM); goto dec_count; } - htab_elem_set_ptr_to_pptr(l_new, ptr); + htab_elem_set_ptr_to_pptr(htab, l_new, ptr); pptr = *(void __percpu **)ptr; } @@ -1241,7 +1347,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); } - l_new->hash = hash; + if (htab_has_hash(htab)) + htab_elem_set_hash(l_new, hash); + else + WRITE_ONCE(*(unsigned long *)l_new->key, + htab_zero_extend_key(key, key_size)); return l_new; dec_count: dec_elem_count(htab); @@ -1292,6 +1402,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, return -EINVAL; /* find an element without taking the bucket lock */ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1313,7 +1424,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1409,7 +1520,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1476,7 +1587,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1550,7 +1661,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1615,7 +1726,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (l) hlist_nulls_del_rcu(&l->hash_node); else @@ -1650,7 +1761,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, true); if (l) hlist_nulls_del_rcu(&l->hash_node); @@ -1791,7 +1902,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (!l) { ret = -ENOENT; goto out_unlock; @@ -2977,18 +3088,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_mem_dtor(void *obj, void *ctx) -{ - struct htab_btf_record *hrec = ctx; - struct rhtab_elem *elem = obj; - - if (IS_ERR_OR_NULL(hrec->record)) - return; - - bpf_obj_free_fields(hrec->record, - rhtab_elem_value(elem, hrec->key_size)); -} - static void rhtab_free_elem(void *ptr, void *arg) { struct bpf_rhtab *rhtab = arg; @@ -3214,7 +3313,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, if (btf_type_is_void(key_type)) return -EINVAL; - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, htab_mem_dtor, + offsetof(struct rhtab_elem, data) + + round_up(map->key_size, 8)); } static void rhtab_map_free_internal_structs(struct bpf_map *map) diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index f71be4fc8dd7..6bd4cb68c20c 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 40); + VERIFY(hash->elem_size == 32); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0); From c87652656a55e5e8cf994364314a75fa95fb2d69 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Mon, 12 Oct 2026 22:01:59 +0800 Subject: [PATCH 515/660] selftests/bpf: Fix prog_tests framework subtest count checks The hardcoded expected skip_cnt/error_cnt in test_prog_tests_framework() mismatch the real count when the binary is run with -t or -n filters, since some subtests are not selected. Count the executed subtests and compare against that value instead. Fixes: 2324257dbd68 ("selftests/bpf: Refactor prog_tests logging and test execution") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140202.2754552-2-kafai.wan@linux.dev --- .../selftests/bpf/prog_tests/prog_tests_framework.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index d111fe105447..53347f379924 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -13,6 +13,7 @@ static void clear_test_state(struct test_state *state) void test_prog_tests_framework(void) { struct test_state *state = env.test_state; + int subtest_cnt; /* in all the ASSERT calls below we need to return on the first * error due to the fact that we are cleaning the test state after @@ -30,25 +31,30 @@ void test_prog_tests_framework(void) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->skip_cnt, 2, "skip_cnt_check")) + if (!ASSERT_EQ(state->skip_cnt, subtest_cnt, "skip_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 3, "subtest_num_check")) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_fail_subtest")) { test__fail(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->error_cnt, 1, "error_cnt_check")) + if (!ASSERT_EQ(state->error_cnt, subtest_cnt, "error_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 4, "subtest_num_check")) return; From cc4b1c161d92992544cc571a2475baa5b072e120 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Mon, 12 Oct 2026 22:02:00 +0800 Subject: [PATCH 516/660] selftests/bpf: Skip expected_msgs subtests when stdout is not saved stdio_hijack_init() does not set subtest_state->stdout_saved when running with -v in non-parallel mode, since output goes straight to stdout. In that case subtest_state->log_buf is left NULL, so the expected-messages subtests crash when inspecting it. Skip them when stdout was not saved. Fixes: 34c513be3dad ("selftests/bpf: __not_msg() tag for test_loader framework") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140202.2754552-3-kafai.wan@linux.dev --- .../testing/selftests/bpf/prog_tests/prog_tests_framework.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index 53347f379924..b4cd7c7ff439 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -169,6 +169,12 @@ void test_prog_tests_framework_expected_msgs(void) for (i = 0; i < ARRAY_SIZE(cases); i++) { if (test__start_subtest(cases[i].name)) { + if (!env.subtest_state->stdout_saved) { + test__skip(); + test__end_subtest(); + continue; + } + error_cnt = env.subtest_state->error_cnt; msgs.patterns = cases[i].pats; msgs.cnt = 0; From 7ebf808caa76c500d52a714f7dc39a1f4952c4d3 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Mon, 12 Oct 2026 22:02:01 +0800 Subject: [PATCH 517/660] selftests/bpf: Skip compare_text subtest when stdout is not saved stdio_hijack_init() does not set subtest_state->stdout_saved when running with -v in non-parallel mode, since output goes straight to stdout. In that case subtest_state->log_buf is left NULL, so the compare_text_mismatch subtest crashes when inspecting it. Skip it when stdout was not saved. Fixes: bd71904697c6 ("selftests/bpf: Add compare_text_to_expected() helper") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012140202.2754552-4-kafai.wan@linux.dev --- .../bpf/prog_tests/prog_tests_framework.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index b4cd7c7ff439..ddb6cdc1579f 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -203,14 +203,18 @@ void test_prog_tests_framework_compare_text(void) } if (test__start_subtest("compare_text_mismatch")) { - err = compare_text_to_expected("line two\n", "line one\n"); - fflush(stdout); + if (!env.subtest_state->stdout_saved) { + test__skip(); + } else { + err = compare_text_to_expected("line two\n", "line one\n"); + fflush(stdout); - ASSERT_EQ(err, -1, "mismatch_rc"); - ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", - "diff_has_expected"); - ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", - "diff_has_actual"); + ASSERT_EQ(err, -1, "mismatch_rc"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", + "diff_has_expected"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", + "diff_has_actual"); + } test__end_subtest(); } } From c961f7b41a108313a4705baac7b27d2174cc8035 Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Mon, 12 Oct 2026 14:19:12 +0800 Subject: [PATCH 518/660] bpf: Fix NULL task dereference in task work cancellation bpf_task_work_irq() publishes ctx->work with task_work_add() before changing the context from SCHEDULING to SCHEDULED. The target task can run the callback on another CPU before the handler reaches its final state transition. The handler then reloads ctx->task for cancellation if its final cmpxchg observes FREED. By that point the callback or context destruction may already have cleared ctx->task, causing task_work_cancel() to dereference a NULL task pointer. One concrete interleaving is: 1. CPU 0 changes PENDING to SCHEDULING and successfully queues ctx->work with task_work_add(), but has not attempted the final state cmpxchg. 2. The target task on CPU 1 runs the callback. It changes SCHEDULING to RUNNING, executes the BPF subprogram, resets ctx->task, publishes STANDBY and drops the callback's context reference. 3. CPU 2 deletes the map value. It changes STANDBY to FREED and drops the map's context reference without queuing asynchronous cancellation. 4. CPU 0 resumes. Its SCHEDULING-to-SCHEDULED cmpxchg observes FREED, and the cancellation attempt passes NULL to task_work_cancel(). If deletion publishes FREED before the callback claims RUNNING, the callback takes its FREED exit and drops its context reference. Context destruction can then clear ctx->task before CPU 0 attempts cancellation. A controlled reproducer with a delay before the final cmpxchg triggered the following fault on a v7.3-rc3 based x86-64 KVM guest: BUG: kernel NULL pointer dereference, address: 0000000000000890 RIP: 0010:task_work_cancel+0xd/0xa0 RDI: 0000000000000000 CR2: 0000000000000890 Call Trace: bpf_task_work_irq+0x95/0x100 irq_work_run_list+0x4f/0x90 irq_work_run+0x18/0x50 0x890 is the task_struct::task_works offset in that build, read through the NULL task argument. Capture ctx->task before task_work_add() publishes the callback and use that pointer for the handler's cancellation attempt. The round still holds its task reference when the pointer is captured. Releasing that reference uses bpf_task_release(), which calls put_task_struct_rcu_user(), so the handler's existing RCU read-side section keeps the captured task alive even if ctx->task is subsequently cleared. The asynchronous cancellation path reads ctx->task directly, which is safe there: queuing it requires deletion to observe SCHEDULED, and that xchg wins against the callback's transition to RUNNING, so the callback takes its FREED exit without resetting ctx->task, while the map reference transferred to the cancellation handler prevents context destruction until cancellation completes. The handler-tail and asynchronous cancellation attempts are mutually exclusive: asynchronous cancellation is queued only if deletion observes SCHEDULED after the scheduling handler's final cmpxchg has succeeded. If that cmpxchg instead observes FREED, the scheduling handler attempts cancellation with the captured pointer. Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs") Reported-by: Jackie Liu Suggested-by: Mykyta Yatsenko Signed-off-by: Yun Lu Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012061915.231254-2-luyun_611@163.com --- kernel/bpf/helpers.c | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index a284f20c97d5..4460624255cd 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4566,15 +4566,21 @@ static void bpf_task_work_ctx_put(struct bpf_task_work_ctx *ctx) } } -static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx) +static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, + struct task_struct *task) { /* * Scheduled task_work callback holds ctx ref, so if we successfully * cancelled, we put that ref on callback's behalf. If we couldn't * cancel, callback will inevitably run or has already completed * running, and it would have taken care of its ctx ref itself. + * + * The caller passes the task explicitly: the scheduling handler must + * not read ctx->task after the work is published (the callback may + * have reset it), while in the asynchronous cancellation path + * ctx->task is stable, see bpf_task_work_cancel_scheduled(). */ - if (task_work_cancel(ctx->task, &ctx->work)) + if (task_work_cancel(task, &ctx->work)) bpf_task_work_ctx_put(ctx); } @@ -4615,6 +4621,7 @@ static void bpf_task_work_callback(struct callback_head *cb) static void bpf_task_work_irq(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); + struct task_struct *task; enum bpf_task_work_state state; int err; @@ -4625,7 +4632,16 @@ static void bpf_task_work_irq(struct irq_work *irq_work) return; } - err = task_work_add(ctx->task, &ctx->work, ctx->mode); + /* + * Capture the task pointer before the work is published: once the + * callback runs, it may reset ctx->task. The capture happens while + * this round's task reference is held, and every path that can + * release it runs after this handler entered its rcu read-side + * section. bpf_task_release() is put_task_struct_rcu_user(), so the + * task_struct itself cannot be freed before this section ends. + */ + task = ctx->task; + err = task_work_add(task, &ctx->work, ctx->mode); if (err) { bpf_task_work_ctx_reset(ctx); /* @@ -4639,14 +4655,14 @@ static void bpf_task_work_irq(struct irq_work *irq_work) /* * It's technically possible for just scheduled task_work callback to - * complete running by now, going SCHEDULING -> RUNNING and then - * dropping its ctx refcount. Instead of capturing an extra ref just - * to protect below ctx->state access, we rely on rcu_read_lock - * above to prevent kfree_rcu from freeing ctx before we return. + * complete running by now, going SCHEDULING -> RUNNING, resetting + * ctx->task and dropping its ctx refcount. The rcu read-side section + * above keeps both the ctx memory and the captured task pointer + * valid until this handler returns. */ state = cmpxchg(&ctx->state, BPF_TW_SCHEDULING, BPF_TW_SCHEDULED); if (state == BPF_TW_FREED) - bpf_task_work_cancel(ctx); /* clean up if we switched into FREED state */ + bpf_task_work_cancel(ctx, task); /* clean up if we switched into FREED state */ } static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *tw, @@ -5006,7 +5022,13 @@ static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); - bpf_task_work_cancel(ctx); /* this might put task_work callback's ref */ + /* + * Deletion observed SCHEDULED and won against the callback's + * transition to RUNNING. The callback takes its FREED exit without + * resetting ctx->task, and the transferred map reference prevents + * destruction until cancellation completes. + */ + bpf_task_work_cancel(ctx, ctx->task); /* this might put task_work callback's ref */ bpf_task_work_ctx_put(ctx); /* and here we put map's own ref that was transferred to us */ } From 6674dcb2785df9f35e7b8234054be2c2bf0c6301 Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Mon, 12 Oct 2026 14:19:13 +0800 Subject: [PATCH 519/660] selftests/bpf: Add task work cancellation race test Add cross-CPU stress coverage for the captured-task cancellation fix. Run scheduling and the target callback on different CPUs, rotating through three map-value lifetime patterns for 1500 rounds. Reuse rounds keep the map value and start the next generation after the callback records its completion. The next scheduling attempt can race with the remaining callback cleanup, checking that the context becomes reusable and subsequent generations make progress. Late-deletion rounds remove the value after the callback records its completion, racing deletion with the remaining callback cleanup and the scheduling handler's final state transition. Early-deletion rounds remove it once the BPF program announces that it is about to schedule, racing deletion with context acquisition and callback publication. These deletion variants stress the cancellation paths fixed by the captured-task change: on an unfixed kernel, a deletion observed by a delayed scheduling handler can route cancellation through an already cleared ctx->task. Publish the scheduling result together with its generation in one packed value, so a stale result cannot be mistaken for the current round's. The BPF program retries transient -EBUSY results without publishing them; in early-deletion rounds, accept -ENOENT when deletion removes the value before a retry looks it up. Check callback completion separately for each generation so that an earlier callback cannot satisfy a later round. This is stress coverage for task-work cancellation races; it does not force the exact faulting interleavings or validate rejection of context reuse while the scheduling irq_work is still BUSY. Signed-off-by: Yun Lu Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012061915.231254-3-luyun_611@163.com --- .../selftests/bpf/prog_tests/test_task_work.c | 281 ++++++++++++++++++ .../selftests/bpf/progs/task_work_race.c | 126 ++++++++ 2 files changed, 407 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/task_work_race.c diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_work.c b/tools/testing/selftests/bpf/prog_tests/test_task_work.c index fe0cb1702eaa..f9fd9adddfe6 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_work.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_work.c @@ -5,10 +5,14 @@ #include #include "task_work.skel.h" #include "task_work_fail.skel.h" +#include "task_work_race.skel.h" #include #include #include #include +#include +#include +#include static int perf_event_open(__u32 type, __u64 config, int pid) { @@ -155,3 +159,280 @@ void test_task_work(void) RUN_TESTS(task_work_fail); } + +#define TASK_WORK_RACE_ROUNDS 1500 +#define TASK_WORK_RACE_POLL_US 100 +#define TASK_WORK_RACE_POLL_ITERS 20000 +#define TASK_WORK_RACE_START_POLL_ITERS 50000 + +/* Must match progs/task_work_race.c. */ +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, +}; + +struct task_work_race_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct task_work_race_ctx { + int stop; + int setup_err; + int trigger_tid; + int target_tid; +}; + +struct task_work_race_thread_arg { + struct task_work_race_ctx *ctx; + int cpu; + int *tid; +}; + +static int task_work_race_pin_cpu(int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +static void *task_work_race_worker(void *arg) +{ + struct task_work_race_thread_arg *thread = arg; + struct task_work_race_ctx *ctx = thread->ctx; + int err; + + err = task_work_race_pin_cpu(thread->cpu); + if (err) + __atomic_store_n(&ctx->setup_err, err, __ATOMIC_RELEASE); + __atomic_store_n(thread->tid, syscall(__NR_gettid), __ATOMIC_RELEASE); + while (!__atomic_load_n(&ctx->stop, __ATOMIC_ACQUIRE)) + getppid(); + return NULL; +} + +static int task_work_race_status(struct task_work_race *skel, int idx, + __s64 *value) +{ + return bpf_map_lookup_elem(bpf_map__fd(skel->maps.status), &idx, + value); +} + +static int task_work_race_set_status(struct task_work_race *skel, int idx, + __s64 value) +{ + return bpf_map_update_elem(bpf_map__fd(skel->maps.status), &idx, + &value, BPF_ANY); +} + +static int task_work_race_wait_ready(struct task_work_race *skel, __u32 seq) +{ + int i; + + for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_race_status(skel, RACE_READY_SEQ, &value) && + value == seq) + return 0; + if (!task_work_race_status(skel, RACE_RESULT, &value) && + (__u32)((__u64)value >> 32) == seq) + /* + * The BPF program finished between the two reads: + * RESULT makes waiting for READY unnecessary. Any error + * that occurred before READY is checked by the caller. + */ + return 0; + usleep(TASK_WORK_RACE_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_prepare_elem(struct task_work_race *skel) +{ + struct task_work_race_value value = {}; + int key = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.hmap), &key, &value)) + return 0; + if (errno != ENOENT) + return -errno; + if (bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return -errno; + return 0; +} + +static int task_work_race_arm(struct task_work_race *skel, __u32 seq) +{ + if (task_work_race_set_status(skel, RACE_ARM_SEQ, seq)) + return -errno; + return task_work_race_wait_ready(skel, seq); +} + +static int task_work_race_check_done(struct task_work_race *skel, __u32 seq) +{ + int i; + + for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { + __s64 result = 0; + + if (!task_work_race_status(skel, RACE_RESULT, &result) && + (__u32)((__u64)result >> 32) == seq) + return (__s32)result; + usleep(TASK_WORK_RACE_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_wait_callback(struct task_work_race *skel, + __u32 seq) +{ + int i, key = seq; + + for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { + __u64 completed = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.completed), + &key, &completed) && completed) + return 0; + usleep(TASK_WORK_RACE_POLL_US); + } + return -ETIMEDOUT; +} + +void serial_test_task_work_race(void) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .tid = &ctx.target_tid, + }; + struct task_work_race *skel; + pthread_t trigger, target; + cpu_set_t allowed; + int cpu, cpu_count = 0; + bool target_started = false, trigger_started = false; + int err, i, key = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_arg.cpu = cpu; + else + target_arg.cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_race__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (!ASSERT_OK(task_work_race__attach(skel), "attach")) + goto cleanup; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (!ASSERT_OK(err, "pthread_create target")) + goto cleanup; + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (!ASSERT_OK(err, "pthread_create trigger")) + goto stop; + trigger_started = true; + + for (i = 0; i < TASK_WORK_RACE_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_RACE_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto stop; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto stop; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + for (i = 0; i < TASK_WORK_RACE_ROUNDS; i++) { + __u32 seq = i + 1; + int variant = i % 3; + + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "prepare elem") || + !ASSERT_OK(task_work_race_arm(skel, seq), "round ready")) + goto stop; + + if (variant == 2 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "early delete")) + goto stop; + + err = task_work_race_check_done(skel, seq); + /* + * Variant 2 deletes the element after READY. A scheduling attempt + * racing with deletion can return -EBUSY, which the BPF program + * retries without publishing RESULT. The retry can then report + * -ENOENT because the element is gone; accept that terminal result. + * A later variant 0 round recreates the element. + */ + if (variant == 2 && err == -ENOENT) + err = 0; + if (!ASSERT_OK(err, "schedule")) { + fprintf(stderr, "round %d schedule failed: %d\n", i, err); + goto stop; + } + + if (variant != 2 && + !ASSERT_OK(task_work_race_wait_callback(skel, seq), + "callback")) + goto stop; + + if (variant == 1 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "late delete")) + goto stop; + } + + /* A distinct completion generation prevents an old callback satisfying this. */ + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "final prepare") || + !ASSERT_OK(task_work_race_arm(skel, TASK_WORK_RACE_ROUNDS + 1), + "final ready") || + !ASSERT_OK(task_work_race_check_done(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final schedule") || + !ASSERT_OK(task_work_race_wait_callback(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final callback")) + goto stop; + +stop: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started) + pthread_join(trigger, NULL); + if (target_started) + pthread_join(target, NULL); +cleanup: + task_work_race__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/task_work_race.c b/tools/testing/selftests/bpf/progs/task_work_race.c new file mode 100644 index 000000000000..c94eca510cb3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_race.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include +#include +#include +#include +#include +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +#define TASK_WORK_RACE_MAX_SEQ 2048 + +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, + RACE_STATUS_MAX, +}; + +int trigger_tid; +int target_tid; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, RACE_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, TASK_WORK_RACE_MAX_SEQ); + __type(key, int); + __type(value, __u64); +} completed SEC(".maps"); + +struct task_work_race_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 4); + __type(key, int); + __type(value, struct task_work_race_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static int process_work(struct bpf_map *map, void *key, void *value) +{ + struct task_work_race_value *work = value; + __u64 *done; + int seq = work->seq; + + if (seq <= 0 || seq >= TASK_WORK_RACE_MAX_SEQ) + return 0; + done = bpf_map_lookup_elem(&completed, &seq); + if (done) + *done = 1; + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int race_sched_work(void *ctx) +{ + struct task_work_race_value *work; + struct task_struct *task; + __s64 *arm, *result; + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + int key = 0, err; + __u32 seq; + + if (tid != trigger_tid) + return 0; + + key = RACE_ARM_SEQ; + arm = bpf_map_lookup_elem(&status, &key); + key = RACE_RESULT; + result = bpf_map_lookup_elem(&status, &key); + if (!arm || !result || *arm <= 0 || + (__u32)((__u64)*result >> 32) == *arm) + return 0; + seq = *arm; + + task = bpf_task_from_pid(target_tid); + if (!task) { + err = -ESRCH; + goto out_done; + } + + key = 0; + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + err = -ENOENT; + goto out_task; + } + + work->seq = seq; + set_status(RACE_READY_SEQ, seq); + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + process_work); + if (err == -EBUSY) { + bpf_task_release(task); + return 0; + } + +out_task: + bpf_task_release(task); +out_done: + /* Publish the sequence and its error in one store. */ + *result = ((__u64)seq << 32) | (__u32)err; + return 0; +} From a36d9afe8d923ca00dd2001409f420202ee672da Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Mon, 12 Oct 2026 14:19:14 +0800 Subject: [PATCH 520/660] bpf: Prevent task work context reuse while irq_work is busy A bpf_task_work context can return to STANDBY before its scheduling irq_work handler returns. Both callback completion and task_work_add() failure can publish STANDBY while the irq_work is still BUSY. A new round can then reinitialize and queue the same irq_work on another CPU while the old handler is still running. One concrete interleaving is: 1. CPU 0 successfully queues ctx->work with task_work_add() and pauses before its SCHEDULING-to-SCHEDULED cmpxchg. 2. The target task on CPU 1 completes the callback, resets ctx->task and publishes STANDBY. 3. CPU 2 starts a new round, changes STANDBY to PENDING and queues the same irq_work. The new handler changes PENDING to SCHEDULING, but task_work_add() returns -ESRCH because its target task has exited. The handler has not yet reset the context or rolled back the state. 4. CPU 0 resumes and changes the new round's SCHEDULING to SCHEDULED, even though no task work was queued for that round. 5. The new handler resets ctx->task, but its SCHEDULING-to-STANDBY cmpxchg fails. The context remains SCHEDULED with a NULL task, and subsequent attempts to acquire this context fail with -EBUSY. 6. Map-value deletion observes SCHEDULED and queues asynchronous cancellation, which passes the NULL ctx->task to task_work_cancel(). Overlapping invocations also share one BUSY bit. An invocation can clear that bit while the other invocation is still running. Initialize the scheduling irq_work when the context is created and stop reinitializing it for each round. After claiming the context with the STANDBY-to-PENDING cmpxchg, check BUSY and reject reuse with -EBUSY if a previous invocation is still in progress. Checking after the cmpxchg orders the BUSY read after the STANDBY publication. No irq_work has been queued for the attempted round at this point, so the rejection can roll PENDING back to STANDBY. Use cmpxchg for that rollback to preserve FREED if a concurrent deletion has published it. Once BUSY clears, scheduling can be retried. Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs") Signed-off-by: Yun Lu Signed-off-by: Alexei Starovoitov Acked-by: Mykyta Yatsenko Link: https://patch.msgid.link/20261012061915.231254-4-luyun_611@163.com --- kernel/bpf/helpers.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 4460624255cd..439370d73cc5 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4682,6 +4682,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t memset(ctx, 0, sizeof(*ctx)); refcount_set(&ctx->refcnt, 1); /* map's own ref */ ctx->state = BPF_TW_STANDBY; + init_irq_work(&ctx->irq_work, bpf_task_work_irq); old_ctx = cmpxchg(&twk->ctx, NULL, ctx); if (old_ctx) { @@ -4724,6 +4725,18 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work bpf_task_work_ctx_put(ctx); return ERR_PTR(-EBUSY); } + /* + * STANDBY can be published before the previous scheduling irq_work + * returns. Claim PENDING before checking BUSY to order the check after + * that publication. No irq_work has been queued for this attempt yet, + * so reject reuse and roll back to STANDBY while BUSY is set. Preserve + * FREED if deletion wins the rollback race. + */ + if (unlikely(irq_work_is_busy(&ctx->irq_work))) { + (void)cmpxchg(&ctx->state, BPF_TW_PENDING, BPF_TW_STANDBY); + bpf_task_work_ctx_put(ctx); + return ERR_PTR(-EBUSY); + } /* * If no process or bpffs is holding a reference to the map, no new callbacks should be @@ -4773,7 +4786,6 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work ctx->map = map; ctx->map_val = (void *)tw - map->record->task_work_off; init_task_work(&ctx->work, bpf_task_work_callback); - init_irq_work(&ctx->irq_work, bpf_task_work_irq); irq_work_queue(&ctx->irq_work); return 0; From 33b2c64f073c08c93433f71f78586e7571ff3a13 Mon Sep 17 00:00:00 2001 From: Yun Lu Date: Mon, 12 Oct 2026 14:19:15 +0800 Subject: [PATCH 521/660] selftests/bpf: Add task work BUSY window test Add a regression test for reusing a bpf_task_work context while its scheduling irq_work is still BUSY. A test that schedules from a thread pinned to one CPU never observes that window: irq_work_queue() self-IPIs the local CPU and the invocation completes before the thread's next syscall. Use an fexit program on task_work_add() to delay the scheduling handler's return with a time- and iteration-bounded busy-wait, after the first-round callback was published and before the handler's tail state transition. The target task attempts to reuse the same context from a later syscall, strictly after its own callback wrapper returned, so the context is STANDBY at that point. A valid window requires the fexit program to observe the attempt's acknowledgment before returning, which proves the attempt ran while the old handler, and thus the BUSY irq_work, was still in flight. With the map and program references held and no competing scheduler, the only possible source of -EBUSY for that attempt is the BUSY check in the acquire path. Check that the attempt is rejected with -EBUSY, that scheduling succeeds on a bounded retry once BUSY clears, and that the second callback runs exactly once. Each attempt uses dedicated workers and a fresh map value. Reset the shared slots before enabling the workers. Stop and join them at the end of the attempt, then wait for their TIDs to disappear from PID lookup before checking the final callback count or starting another attempt. join alone can return before exit_task_work(); a signal-zero tgkill probe still finds the worker until after its exit task_work has run. Bound that wait and fail on unexpected probe errors. Callback deliveries are also matched against the window's generation. Missed windows are counted and retried within a bounded attempt budget; hard failures fail the test. Record the busy-wait elapsed time after worker cleanup as a diagnostic, without asserting a wall-clock bound, since host vCPU preemption can stretch wall-clock time without affecting the gate behavior under test. Skip when fewer than two CPUs are available or when the loaded kconfig extern identifies CONFIG_PREEMPT_RT. Check that extern before attaching programs or starting workers. Load and attach errors fail the test instead of being inferred from tracefs file paths and reported as unsupported. Signed-off-by: Yun Lu Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261012061915.231254-5-luyun_611@163.com --- .../selftests/bpf/prog_tests/test_task_work.c | 477 ++++++++++++++++-- .../selftests/bpf/progs/task_work_busy.c | 325 ++++++++++++ 2 files changed, 770 insertions(+), 32 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/task_work_busy.c diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_work.c b/tools/testing/selftests/bpf/prog_tests/test_task_work.c index f9fd9adddfe6..ca426094e9c4 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_work.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_work.c @@ -6,6 +6,7 @@ #include "task_work.skel.h" #include "task_work_fail.skel.h" #include "task_work_race.skel.h" +#include "task_work_busy.skel.h" #include #include #include @@ -161,9 +162,9 @@ void test_task_work(void) } #define TASK_WORK_RACE_ROUNDS 1500 -#define TASK_WORK_RACE_POLL_US 100 -#define TASK_WORK_RACE_POLL_ITERS 20000 -#define TASK_WORK_RACE_START_POLL_ITERS 50000 +#define TASK_WORK_POLL_US 100 +#define TASK_WORK_POLL_ITERS 20000 +#define TASK_WORK_START_POLL_ITERS 50000 /* Must match progs/task_work_race.c. */ enum task_work_race_status { @@ -173,7 +174,7 @@ enum task_work_race_status { RACE_RESULT, }; -struct task_work_race_value { +struct task_work_value { __u32 seq; char data[60]; struct bpf_task_work tw; @@ -216,31 +217,46 @@ static void *task_work_race_worker(void *arg) return NULL; } -static int task_work_race_status(struct task_work_race *skel, int idx, - __s64 *value) +static int task_work_status(struct bpf_map *map, int idx, __s64 *value) { - return bpf_map_lookup_elem(bpf_map__fd(skel->maps.status), &idx, - value); + return bpf_map_lookup_elem(bpf_map__fd(map), &idx, value); } -static int task_work_race_set_status(struct task_work_race *skel, int idx, - __s64 value) +static int task_work_set_status(struct bpf_map *map, int idx, __s64 value) { - return bpf_map_update_elem(bpf_map__fd(skel->maps.status), &idx, - &value, BPF_ANY); + return bpf_map_update_elem(bpf_map__fd(map), &idx, &value, BPF_ANY); +} + +/* Wait for seq, keeping polling errors separate from the scheduling result. */ +static int task_work_wait_seq(struct bpf_map *map, int idx, __u32 seq, + __s32 *result) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(map, idx, &value) && + (__u32)((__u64)value >> 32) == seq) { + *result = (__s32)value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; } static int task_work_race_wait_ready(struct task_work_race *skel, __u32 seq) { int i; - for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { __s64 value = 0; - if (!task_work_race_status(skel, RACE_READY_SEQ, &value) && + if (!task_work_status(skel->maps.status, RACE_READY_SEQ, &value) && value == seq) return 0; - if (!task_work_race_status(skel, RACE_RESULT, &value) && + if (!task_work_status(skel->maps.status, RACE_RESULT, &value) && (__u32)((__u64)value >> 32) == seq) /* * The BPF program finished between the two reads: @@ -248,14 +264,14 @@ static int task_work_race_wait_ready(struct task_work_race *skel, __u32 seq) * that occurred before READY is checked by the caller. */ return 0; - usleep(TASK_WORK_RACE_POLL_US); + usleep(TASK_WORK_POLL_US); } return -ETIMEDOUT; } static int task_work_race_prepare_elem(struct task_work_race *skel) { - struct task_work_race_value value = {}; + struct task_work_value value = {}; int key = 0; if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.hmap), &key, &value)) @@ -270,24 +286,18 @@ static int task_work_race_prepare_elem(struct task_work_race *skel) static int task_work_race_arm(struct task_work_race *skel, __u32 seq) { - if (task_work_race_set_status(skel, RACE_ARM_SEQ, seq)) + if (task_work_set_status(skel->maps.status, RACE_ARM_SEQ, seq)) return -errno; return task_work_race_wait_ready(skel, seq); } static int task_work_race_check_done(struct task_work_race *skel, __u32 seq) { - int i; + __s32 result; + int err; - for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { - __s64 result = 0; - - if (!task_work_race_status(skel, RACE_RESULT, &result) && - (__u32)((__u64)result >> 32) == seq) - return (__s32)result; - usleep(TASK_WORK_RACE_POLL_US); - } - return -ETIMEDOUT; + err = task_work_wait_seq(skel->maps.status, RACE_RESULT, seq, &result); + return err ?: result; } static int task_work_race_wait_callback(struct task_work_race *skel, @@ -295,13 +305,13 @@ static int task_work_race_wait_callback(struct task_work_race *skel, { int i, key = seq; - for (i = 0; i < TASK_WORK_RACE_POLL_ITERS; i++) { + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { __u64 completed = 0; if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.completed), &key, &completed) && completed) return 0; - usleep(TASK_WORK_RACE_POLL_US); + usleep(TASK_WORK_POLL_US); } return -ETIMEDOUT; } @@ -359,11 +369,11 @@ void serial_test_task_work_race(void) goto stop; trigger_started = true; - for (i = 0; i < TASK_WORK_RACE_START_POLL_ITERS; i++) { + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) break; - usleep(TASK_WORK_RACE_POLL_US); + usleep(TASK_WORK_POLL_US); } if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), @@ -436,3 +446,406 @@ stop: cleanup: task_work_race__destroy(skel); } + +#define TASK_WORK_BUSY_ROUNDS 32 +#define TASK_WORK_BUSY_MAX_ATTEMPTS 96 + +/* Must match progs/task_work_busy.c. */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + /* + * High 32 bits hold the sequence. BUSY_WINDOW low bits: 1 open, + * 2 the in-window attempt was seen before the handler returned, + * 3 the wait bound was reached without it. Result slots: signed + * error in the low 32 bits. + */ + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +enum task_work_busy_attempt { + TWB_VALID, + TWB_MISSED, + TWB_HARD_FAIL, +}; + +static const char *twb_attempt_name(enum task_work_busy_attempt r) +{ + switch (r) { + case TWB_VALID: + return "valid"; + case TWB_MISSED: + return "missed"; + default: + return "fail"; + } +} + +static int task_work_busy_wait_value(struct task_work_busy *skel, int idx, + __s64 want) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, idx, &value) && value == want) + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* Wait until the window slot reports a final state (2 or 3) for seq. */ +static int task_work_busy_wait_window(struct task_work_busy *skel, __u32 seq, + __s64 *window) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, BUSY_WINDOW, &value) && + (__u32)((__u64)value >> 32) == seq && + (__u32)value >= 2) { + *window = value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_busy_prepare_elem(struct task_work_busy *skel, int key) +{ + struct task_work_value value = {}; + + if (!bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return 0; + return -errno; +} + +static int task_work_busy_reset(struct task_work_busy *skel) +{ + static const int zeroed[] = { + BUSY_WINDOW, BUSY_ACK, BUSY_GEN1_RESULT, BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, BUSY_CB2_COUNT, BUSY_CB2_SEQ, BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(zeroed); i++) + if (task_work_set_status(skel->maps.status, zeroed[i], 0)) + return -errno; + return 0; +} + +/* + * join waits for clear_child_tid, which is cleared before exit_task_work(). + * PID lookup stops finding the worker only after its exit task_work has run. + * Probe with signal 0; no signal is delivered. Do not create new workers + * until both old TIDs have disappeared. + */ +static int task_work_busy_wait_exit(int tid) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + if (syscall(__NR_tgkill, getpid(), tid, 0)) { + if (errno == ESRCH) + return 0; + return -errno; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* + * Run one attempt on dedicated workers and report its outcome. The + * workers are stopped, joined and waited out of PID lookup before another + * attempt can reset the shared state. join alone does not wait for their + * exit task_work to finish. + */ +static enum task_work_busy_attempt +task_work_busy_run_attempt(struct task_work_busy *skel, int seq, + int trigger_cpu, int target_cpu, __s64 *out_wait_ns) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .cpu = trigger_cpu, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .cpu = target_cpu, + .tid = &ctx.target_tid, + }; + enum task_work_busy_attempt result = TWB_HARD_FAIL; + pthread_t trigger, target; + bool trigger_started = false, target_started = false; + bool cleanup_ok = true; + __s64 window = 0, wait_ns = 0, count = -1; + __s32 res = 0; + int err, i; + + *out_wait_ns = 0; + /* Do not let a reused TID match the previous attempt's filter. */ + skel->bss->trigger_tid = 0; + skel->bss->target_tid = 0; + + /* Clear old state before enabling workers that can read it. */ + if (!ASSERT_OK(task_work_busy_prepare_elem(skel, seq), "prepare elem")) + goto out; + if (!ASSERT_OK(task_work_busy_reset(skel), "reset slots")) + goto out; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (err) { + ASSERT_OK(err, "pthread_create target"); + goto out; + } + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (err) { + ASSERT_OK(err, "pthread_create trigger"); + goto out; + } + trigger_started = true; + + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto out; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto out; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + /* Arm last, once this attempt's value, slots and TIDs are in place. */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_ARM_SEQ, seq), + "arm")) + goto out; + + /* A fresh value has no expected BUSY source: check the first result. */ + err = task_work_wait_seq(skel->maps.status, BUSY_GEN1_RESULT, seq, &res); + if (!ASSERT_OK(err, "gen1 result")) + goto out; + if (!ASSERT_OK(res, "gen1 schedule")) { + fprintf(stderr, "attempt %d: gen1 failed: %d\n", seq, res); + goto out; + } + + if (!ASSERT_OK(task_work_busy_wait_window(skel, seq, &window), + "window close")) + goto out; + + if ((__u32)window == 3) { + /* + * The attempt did not arrive inside the wait bound. The + * elapsed time is a diagnostic only; count + * the miss and retry with a fresh attempt. + */ + result = TWB_MISSED; + goto out; + } + + /* + * Valid window: the handler saw this sequence's attempt before + * returning, so the attempt ran while the scheduling irq_work was + * still BUSY. The elapsed time is a diagnostic only: host vCPU + * preemption can stretch wall-clock time without affecting the + * gate behavior under test. + */ + err = task_work_wait_seq(skel->maps.status, BUSY_ACK, seq, &res); + if (!ASSERT_OK(err, "in-window attempt")) + goto out; + if (res != -EBUSY) { + fprintf(stderr, + "attempt %d: in-window reuse %s (%d), expected -EBUSY\n", + seq, res == 0 ? "accepted" : "failed", res); + ASSERT_EQ(res, -EBUSY, "in-window BUSY gate rejection"); + goto out; + } + + /* + * The window was released; retry until BUSY clears. The second + * callback must then run exactly once. + */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_RECOVER_SEQ, seq), + "enable recovery")) + goto out; + err = task_work_wait_seq(skel->maps.status, BUSY_GEN2_RESULT, seq, &res); + if (!ASSERT_OK(err, "recovery result")) + goto out; + if (!ASSERT_OK(res, "recovery schedule")) { + fprintf(stderr, "attempt %d: recovery failed: %d\n", seq, res); + goto out; + } + if (!ASSERT_OK(task_work_busy_wait_value(skel, BUSY_CB2_SEQ, seq), + "gen2 callback")) + goto out; + result = TWB_VALID; +out: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started && + (!ASSERT_OK(pthread_join(trigger, NULL), "trigger join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.trigger_tid), "trigger exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + if (target_started && + (!ASSERT_OK(pthread_join(target, NULL), "target join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.target_tid), "target exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + /* Both workers have passed exit_task_work; the count is now stable. */ + if (result == TWB_VALID) { + if (!ASSERT_OK(task_work_status(skel->maps.status, BUSY_CB2_COUNT, &count), + "cb2 count")) { + result = TWB_HARD_FAIL; + } else if (!ASSERT_EQ(count, 1, "gen2 callback exactly once")) { + fprintf(stderr, "attempt %d: gen2 callback count %lld\n", + seq, (long long)count); + result = TWB_HARD_FAIL; + } + } + /* + * Only a captured final window belongs to this attempt. The fexit + * runs in the trigger's interrupted context, so successful worker + * cleanup also completes its diagnostic write. Read once here; + * the elapsed value does not decide whether the attempt passed. + */ + if (cleanup_ok && window) { + err = task_work_status(skel->maps.status, BUSY_LAST_WAIT_NS, &wait_ns); + if (err) { + err = -errno; + if (result != TWB_HARD_FAIL) { + ASSERT_OK(err, "wait diagnostic"); + result = TWB_HARD_FAIL; + } else { + fprintf(stderr, + "attempt %d: elapsed diagnostic unavailable: %d\n", + seq, err); + } + } + } + *out_wait_ns = wait_ns; + fprintf(stderr, "attempt %d: %s (window %u wait_ns %lld)\n", + seq, twb_attempt_name(result), (__u32)window, (long long)wait_ns); + return result; +} + +void serial_test_task_work_busy(void) +{ + struct task_work_busy *skel; + struct bpf_link *link_tp = NULL, *link_fexit = NULL; + cpu_set_t allowed; + int cpu, cpu_count = 0, trigger_cpu = -1, target_cpu = -1; + int attempt, attempts_started = 0, valid = 0, missed = 0; + bool hard_fail = false; + __s64 max_wait_ns = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_cpu = cpu; + else + target_cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_busy__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (skel->kconfig->CONFIG_PREEMPT_RT) { + printf("%s:SKIP:CONFIG_PREEMPT_RT=y runs irq_work in a kernel thread\n", + __func__); + test__skip(); + goto cleanup; + } + + link_tp = bpf_program__attach_tracepoint(skel->progs.busy_sched_work, + "syscalls", + "sys_enter_getppid"); + if (!ASSERT_OK_PTR(link_tp, "attach tracepoint")) + goto cleanup; + + link_fexit = bpf_program__attach_trace(skel->progs.busy_fexit_task_work_add); + if (!ASSERT_OK_PTR(link_fexit, "attach fexit")) + goto cleanup; + + for (attempt = 1; + attempt <= TASK_WORK_BUSY_MAX_ATTEMPTS && + valid < TASK_WORK_BUSY_ROUNDS; + attempt++) { + enum task_work_busy_attempt res; + __s64 wait_ns = 0; + + attempts_started = attempt; + res = task_work_busy_run_attempt(skel, attempt, trigger_cpu, + target_cpu, &wait_ns); + switch (res) { + case TWB_VALID: + valid++; + if (wait_ns > max_wait_ns) + max_wait_ns = wait_ns; + break; + case TWB_MISSED: + missed++; + break; + case TWB_HARD_FAIL: + hard_fail = true; + break; + } + if (hard_fail) + break; + } + + fprintf(stderr, + "task_work_busy: %d/%d valid, %d missed, %d attempts, max_wait %lld ns\n", + valid, TASK_WORK_BUSY_ROUNDS, missed, attempts_started, + (long long)max_wait_ns); + + /* + * The coverage assertion only makes sense when no hard + * failure aborted the loop; after a hard failure keep the primary + * reason instead of deriving more assertion noise from it. + */ + if (!hard_fail) + ASSERT_EQ(valid, TASK_WORK_BUSY_ROUNDS, "valid busy windows"); +cleanup: + bpf_link__destroy(link_tp); + bpf_link__destroy(link_fexit); + task_work_busy__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/task_work_busy.c b/tools/testing/selftests/bpf/progs/task_work_busy.c new file mode 100644 index 000000000000..9960011edadc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_busy.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include +#include +#include +#include +#include +#include + +char _license[] SEC("license") = "GPL"; + +extern bool CONFIG_PREEMPT_RT __kconfig __weak; + +/* + * Bounds for the hardirq busy-wait that holds the scheduling handler + * inside the real window: task_work_add() has published the callback, + * but the handler's tail state transition has not run yet. + * irq_work_single() clears BUSY only after the handler returns, so + * while this wait runs, the context's scheduling irq_work is BUSY, + * even though the first callback may already have completed and + * returned the context to STANDBY. + * + * The wait is bounded both by a bpf_ktime_get_ns() check and by an + * iteration budget. The time bound is checked against a monotonic + * clock: host vCPU preemption can stretch the wall-clock duration of + * the window, which only makes the window longer, never shorter. + */ +#define BUSY_WAIT_NS 4000000ULL +#define BUSY_WAIT_ITERS 1000000ULL + +int trigger_tid; +int target_tid; + +/* + * High 32 bits of the result slots hold the sequence, low 32 bits the + * signed error. BUSY_WINDOW low bits: 1 open, 2 the in-window attempt + * was seen before the handler returned, 3 the wait bound was reached + * without it. Must match the test. + */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, BUSY_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct task_work_busy_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +/* + * max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each + * attempt schedules into its own fresh value. + */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 128); + __type(key, int); + __type(value, struct task_work_busy_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static __always_inline __s64 get_status(int key) +{ + __s64 *slot = bpf_map_lookup_elem(&status, &key); + + return slot ? *slot : 0; +} + +struct busy_wait_ctx { + __u64 seq; + __u64 start; + bool acked; +}; + +/* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */ +static long busy_wait_cb(__u32 i, void *pctx) +{ + struct busy_wait_ctx *w = pctx; + + if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) { + w->acked = true; + return 1; + } + if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS) + return 1; + return 0; +} + +static int busy_cb_gen1(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __u64 we = (__u64)get_status(BUSY_WINDOW) >> 32; + + /* + * Backstop for deliveries from abandoned attempts: accept only + * while no window for a different sequence is published. The + * test waits for the old workers to disappear from PID lookup, + * after their exit task_work, before resetting the shared state. + * This callback may run before fexit publishes the window state. + */ + if (we && we != (__u64)work->seq) + return 0; + set_status(BUSY_CB1_SEQ, work->seq); + return 0; +} + +static int busy_cb_gen2(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __s64 *count; + int key2 = BUSY_CB2_COUNT; + + /* + * Count only deliveries belonging to the sequence whose window is + * (or was) published: a late delivery from an abandoned attempt + * must not satisfy a later exactly-once check. + */ + if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32)) + return 0; + set_status(BUSY_CB2_SEQ, work->seq); + count = bpf_map_lookup_elem(&status, &key2); + if (count) + __sync_fetch_and_add(count, 1); + return 0; +} + +SEC("fexit/task_work_add") +int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task, + struct callback_head *work, int notify, int retval) +{ + struct busy_wait_ctx w = {}; + __u64 seq, elapsed; + + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq) + return 0; + + /* + * One-shot arm. Only this test's own first-round add may consume + * it and hold the window: a successful TWA_SIGNAL add of the + * bpf_task_work callback, issued from the trigger thread's CPU by + * the self-IPI'd scheduling handler and targeting the target + * thread. Everything else leaves the arm untouched. + */ + if (retval != 0 || notify != TWA_SIGNAL || + BPF_CORE_READ(task, pid) != target_tid) + return 0; + if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid) + return 0; + + set_status(BUSY_ARM_SEQ, 0); + set_status(BUSY_WINDOW, (seq << 32) | 1); + + w.seq = seq; + w.start = bpf_ktime_get_ns(); + bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0); + elapsed = bpf_ktime_get_ns() - w.start; + + /* + * Publish elapsed time for logging after worker cleanup. Window + * validity depends on seeing ACK here, not on this diagnostic. + */ + set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed); + if (w.acked) + set_status(BUSY_WINDOW, (seq << 32) | 2); + else + set_status(BUSY_WINDOW, (seq << 32) | 3); + return 0; +} + +/* + * Attempt the second scheduling round on the same context, targeting the + * trigger thread. The attempt uses the value keyed by its own sequence, + * so it can never touch another attempt's context. Returns true if a + * terminal result was published into res_key; a transient -EBUSY is left + * unpublished for the caller's retry policy, mirroring the race test's + * convention. + */ +static __always_inline bool busy_try_gen2(__u64 seq, int res_key) +{ + struct task_work_busy_value *work; + struct task_struct *task; + int key = (int)seq, err; + + task = bpf_task_from_pid(trigger_tid); + if (!task) { + set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH); + return true; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT); + return true; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen2); + bpf_task_release(task); + if (err == -EBUSY) + return false; + set_status(res_key, ((__s64)seq << 32) | (__u32)err); + return true; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int busy_sched_work(void *ctx) +{ + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + struct task_work_busy_value *work; + struct task_struct *task; + __u64 window, seq; + int key, err; + + /* Also keep the kconfig extern in object BTF for the userspace skip. */ + if (CONFIG_PREEMPT_RT) + return 0; + + if (tid == (__u32)trigger_tid) { + /* + * Trigger thread: schedule the first round once per armed + * sequence, targeting the target thread. The kfunc queues + * the scheduling irq_work and returns; the handler then + * parks inside the fexit window above. A fresh value has no + * expected BUSY source, so publish every scheduling result. + */ + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq || + ((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq) + return 0; + key = (int)seq; + task = bpf_task_from_pid(target_tid); + if (!task) { + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ESRCH); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ENOENT); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen1); + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err); + return 0; + } + + if (tid != (__u32)target_tid) + return 0; + + window = (__u64)get_status(BUSY_WINDOW); + seq = window >> 32; + if (!seq || (__u32)window == 0) + return 0; + /* + * The first callback must have completed for this sequence. Being + * inside the target thread's own subsequent syscall proves the + * callback wrapper has returned and the context is STANDBY, which + * is stronger than observing the completion marker from another + * thread. + */ + if ((__u64)get_status(BUSY_CB1_SEQ) != seq) + return 0; + + if ((__u64)get_status(BUSY_ACK) >> 32 != seq) { + /* + * The in-window attempt, exactly once per sequence. On a + * fixed kernel it must be rejected with -EBUSY: the context + * is STANDBY, the map and program references are alive and + * no other scheduler exists, so only the BUSY gate can + * reject here. + */ + if (!busy_try_gen2(seq, BUSY_ACK)) + set_status(BUSY_ACK, + ((__s64)seq << 32) | (__u32)-EBUSY); + return 0; + } + + /* + * Recovery, gated by the test: the fexit program releasing the + * window does not mean the whole old handler has returned and + * BUSY has cleared yet, so retry until the schedule succeeds. + */ + if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq) + return 0; + if ((__s32)get_status(BUSY_ACK) == 0) + return 0; /* scheduled in-window; nothing to recover */ + if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq) + return 0; /* terminal recovery result published */ + busy_try_gen2(seq, BUSY_GEN2_RESULT); + return 0; +} From a3923834d155ce7c3f630d891564791eeba8bb26 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 14 Oct 2026 21:01:48 -0700 Subject: [PATCH 522/660] bpf: Pack bpf_insn_aux_data flags into bit fields struct bpf_insn_aux_data spreads its flags over eleven bools, a u8 of three bit fields and a u32 group with 25 of its 32 bits spare. Put them all in u64 bit fields, alu_state with them, which fit in the 32 bits beside seen, so orig_idx starts the next word. The structure goes from 216 bytes to 200, and stays there when a later patch adds a u32, which takes a hole further down. alu_state was a u8, but it only ever holds BPF_ALU_* flags, the highest of which is BPF_ALU_IMMEDIATE (1U << 4), so it gets five bits. With eight, the flags would take 34 bits, more than fit beside seen, so they would start a word of their own, and the later u32 would grow the structure to 208 bytes. A new BPF_ALU_* flag has to widen it. No functional change. Suggested-by: Eduard Zingerman Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 49 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 791398e14962..112ca1511da4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -767,44 +767,45 @@ struct bpf_insn_aux_data { u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter__next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ - bool bb_end; /* last instruction of a basic block */ - bool need_scev; - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter__next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ - u32 non_stack_access:1; /* instruction can access non-stack memory */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. From 3c824dfd17e6e918e45a6a96bda0d020ddec8287 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 10:04:37 +0800 Subject: [PATCH 523/660] selftests/bpf: Keep the bpffs of cgroup_hierarchical_stats private setup_hierarchy() mounts a bpffs on /sys/fs/bpf with setup_bpffs(), and then sets up the cgroups with setup_cgroups(), which moves the test into a private mount namespace. The bpffs thus lands in the mount namespace that test_progs shares with the tests running in parallel, on top of the bpffs already there, and hides what they pin. cleanup_bpffs() later runs in the private mount namespace and can't unmount it, so it stays there for good. bpftool_maps_access then fails to pin an iterator because /sys/fs/bpf/test_bpftool_map doesn't exist anymore, and bpftool_metadata can't find the program it pinned in /sys/fs/bpf/test_metadata. Set up the cgroups first, so that the bpffs is mounted in the private mount namespace. Fixes: 88886309d2e8 ("selftests/bpf: add a selftest for cgroup hierarchical stats collection") Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009-selftests-bpf-cgroup-stats-bpffs-v2-1-f93a111da992@gmail.com --- .../selftests/bpf/prog_tests/cgroup_hierarchical_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c index 3bd27d2ea668..ad19d6d41918 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c @@ -151,7 +151,7 @@ static void cleanup_cgroups(void) /* Sets up cgroup hiearchary, returns 0 on success. */ static int setup_hierarchy(void) { - return setup_bpffs() || setup_cgroups(); + return setup_cgroups() || setup_bpffs(); } static void destroy_hierarchy(void) From e0645cfd0e1b2ae7b8ad989bef00a30016b16b5f Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 10:04:38 +0800 Subject: [PATCH 524/660] selftests/bpf: Unmount the bpffs that cgroup_hierarchical_stats mounted setup_bpffs() mounts a bpffs on /sys/fs/bpf and records in mounted_bpffs whether it did, so that cleanup_bpffs() unmounts only a bpffs that the test mounted itself. cleanup_bpffs() checks the flag the other way around. As mounting on /sys/fs/bpf succeeds even when a bpffs is already there, the test never unmounts the bpffs it mounted, and every run leaves another one stacked on /sys/fs/bpf of the mount namespace that the test_progs worker keeps running later tests in. Unmount the bpffs only when setup_bpffs() mounted it. Fixes: 88886309d2e8 ("selftests/bpf: add a selftest for cgroup hierarchical stats collection") Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Reviewed-by: Yosry Ahmed Link: https://patch.msgid.link/20261009-selftests-bpf-cgroup-stats-bpffs-v2-2-f93a111da992@gmail.com --- .../selftests/bpf/prog_tests/cgroup_hierarchical_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c index ad19d6d41918..cc6ab2c263da 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c @@ -110,7 +110,7 @@ static void cleanup_bpffs(void) ASSERT_OK(rmdir(BPFFS_ATTACH_COUNTERS), "rmdir "BPFFS_ATTACH_COUNTERS); /* Unmount bpffs, if it wasn't already mounted when we started */ - if (mounted_bpffs) + if (!mounted_bpffs) return; ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs"); From 17f6fc3ad257b0ad7a11f276a5c0f57e0bdf3f4a Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 16 Oct 2026 11:25:34 +0800 Subject: [PATCH 525/660] selftests/bpf: Record only the test ICMP echo in tc8 of tc_netkit_scrub tc_netkit_scrub attaches tc8 to the primary netkit device, sends an ICMP echo with a mark and a priority through it, and checks the mark and the priority that tc8 saw against what the scrub mode lets through. tc8 records the mark and priority of every packet it sees, and the device also carries what the system sends on its own once the device is up, like IPv6 neighbor discovery or the mDNS of avahi-daemon. When such a packet goes out after the echo, it overwrites what tc8 recorded, and the test finds a mark and a priority of 0. Record only IPv4 ICMP packets in tc8. tc_netkit_scrub run 100 times on a bpf-next kernel with KASAN enabled, x86_64 VM with 32 vCPUs and avahi-daemon running, before and after this patch: before after outcome 2 of 100 failed (mark and priority 0) 0 of 100 failed Fixes: 716fa7dadf11 ("selftests/bpf: Extend netkit tests to validate skb meta data") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261016-selftests-bpf-tc-netkit-scrub-v2-1-b484a7ce2a63@gmail.com --- tools/testing/selftests/bpf/progs/test_tc_link.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/test_tc_link.c b/tools/testing/selftests/bpf/progs/test_tc_link.c index 630f12e51b07..2a91793674dd 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_link.c +++ b/tools/testing/selftests/bpf/progs/test_tc_link.c @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include #include @@ -119,11 +121,19 @@ SEC("tc/egress") int tc8(struct __sk_buff *skb) { struct net_device *dev = BPF_CORE_READ((struct sk_buff *)skb, dev); + struct iphdr iph = {}; + if (skb->protocol != __bpf_constant_htons(ETH_P_IP)) + goto out; + if (bpf_skb_load_bytes(skb, sizeof(struct ethhdr), &iph, sizeof(iph))) + goto out; + if (iph.protocol != IPPROTO_ICMP) + goto out; seen_tc8 = true; mark = skb->mark; prio = skb->priority; headroom = BPF_CORE_READ(dev, needed_headroom); tailroom = BPF_CORE_READ(dev, needed_tailroom); +out: return TCX_PASS; } From 34d527e513c9a3f3a00f1ddf03fe06f493da14d0 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 09:49:00 +0800 Subject: [PATCH 526/660] selftests/bpf: Expect at least one kprobe miss in missed/perf_kprobe missed_kprobe attaches test1 to bpf_fentry_test1 and test2 to bpf_kfunc_common_test, which test1 calls. test_missed_perf_kprobe() expects the kprobe of test2 to miss exactly one run, the one test1 triggers, since kprobes don't nest. The test is not serial, and the kprobe on bpf_fentry_test1 is global. Any other test that does bpf_prog_test_run() on a fentry/fexit program calls bpf_fentry_test1() and thereby fires test1, which calls bpf_kfunc_common_test() from kprobe context. The kprobe of test2 then counts an extra miss, and the test reads 2 instead of 1. Expect at least one miss, as missed_kprobe_recursion does since commit 08a7078feacf ("selftests/bpf: Improve missed_kprobe_recursion test robustness"). Fixes: 01e4ae474e39 ("selftests/bpf: Add test for missed counts of perf event link kprobe") Signed-off-by: Qiliang Yuan Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261009-selftests-bpf-missed-perf-kprobe-v1-1-c9d0f75f6f99@gmail.com --- tools/testing/selftests/bpf/prog_tests/missed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/missed.c b/tools/testing/selftests/bpf/prog_tests/missed.c index 0b8f1aa5a3e7..b5fb0267e944 100644 --- a/tools/testing/selftests/bpf/prog_tests/missed.c +++ b/tools/testing/selftests/bpf/prog_tests/missed.c @@ -36,7 +36,7 @@ static void test_missed_perf_kprobe(void) ASSERT_EQ(info.type, BPF_LINK_TYPE_PERF_EVENT, "info.type"); ASSERT_EQ(info.perf_event.type, BPF_PERF_EVENT_KPROBE, "info.perf_event.type"); - ASSERT_EQ(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); + ASSERT_GE(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); cleanup: missed_kprobe__destroy(skel); From 21c755efaab1f40c2588ae5acbfa319a6d7d3638 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 16 Oct 2026 09:50:57 +0800 Subject: [PATCH 527/660] bpftool: Fix inverted tracefs validation when scanning /proc/mounts The /proc/mounts fallback in get_tracefs_pipe() accepts only non-tracefs mount points, so a tracefs mounted outside the default locations is ignored: bpftool prog tracelog then mounts a second tracefs, or exits silently with --nomount. Negate the condition. Fixes: be3245e22d22 ("tools: bpftool: attempt to mount tracefs if required for tracelog cmd") Signed-off-by: Yuan Chen Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261016015057.127061-1-chenyuan_fl@163.com --- tools/bpf/bpftool/tracelog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/tracelog.c b/tools/bpf/bpftool/tracelog.c index 573a8d99f009..e104748c8350 100644 --- a/tools/bpf/bpftool/tracelog.c +++ b/tools/bpf/bpftool/tracelog.c @@ -86,7 +86,7 @@ static bool get_tracefs_pipe(char *mnt) fclose(fp); /* The string from fscanf() might be truncated, check mnt is valid */ - if (found && validate_tracefs_mnt(mnt, TRACEFS_MAGIC)) + if (found && !validate_tracefs_mnt(mnt, TRACEFS_MAGIC)) goto exit_found; if (block_mount) From 760c953525bab88f053e61f1c1a6b2a5f76911c0 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 12 Oct 2026 22:23:34 +0100 Subject: [PATCH 528/660] selftests/bpf: Fix spelling mistakes in descr literal strings There are spelling mistakes in the descr literal strings. Fix them. Fixes: 095ddb501b39 ("selftests/bpf: Add sockopt case to verify prog_type") Signed-off-by: Colin Ian King Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261012212334.183116-1-colin.i.king@gmail.com --- tools/testing/selftests/bpf/prog_tests/sockopt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index 6c96f2d9fccf..7dfdc4c12dec 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -955,7 +955,7 @@ static struct sockopt_test { /* ==================== prog_type ==================== */ { - .descr = "can attach only BPF_CGROUP_SETSOCKOP", + .descr = "can attach only BPF_CGROUP_SETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1), @@ -969,7 +969,7 @@ static struct sockopt_test { }, { - .descr = "can attach only BPF_CGROUP_GETSOCKOP", + .descr = "can attach only BPF_CGROUP_GETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1), From 09ec92d707236e4af9484092e470078c238531d5 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Fri, 16 Oct 2026 22:20:13 +0800 Subject: [PATCH 529/660] bpf: Initialize ftrace_managed in bpf_trampoline_get() A trampoline managed by ftrace is determined by its target, that said ftrace_managed is already known when the trampoline is initialized. so, set 'ftrace_managed = tr->ip != 0' in bpf_trampoline_get() instead of relying on the first regular or tracing_multi attachment to set it. This makes the backend selection independent of attachment order and keeps the registration callbacks focused on installing the trampoline. No functional change intended. Signed-off-by: Leon Hwang Signed-off-by: Alexei Starovoitov Acked-by: Jiri Olsa Link: https://patch.msgid.link/20261016142014.12420-2-leon.hwang@linux.dev --- kernel/bpf/trampoline.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 24edb59238f3..17f2a8133f61 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -475,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -1286,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1649,7 +1643,6 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; From b443a4578a0d14ab07802aea4bcd93da1c31a592 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Fri, 16 Oct 2026 22:20:14 +0800 Subject: [PATCH 530/660] bpf: Drop unnecessary ftrace_location() in register/unregister/modify_fentry_multi() Since 'tr->ip' is already a 'ftrace_location()'-ed IP in bpf_trampoline_lookup(), it is unnecessary to call ftrace_location() again in register/unregister/modify_fentry_multi(). No functional changes intended. Signed-off-by: Leon Hwang Signed-off-by: Alexei Starovoitov Acked-by: Jiri Olsa Link: https://patch.msgid.link/20261016142014.12420-3-leon.hwang@linux.dev --- kernel/bpf/trampoline.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 17f2a8133f61..21d460866dfe 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1629,8 +1629,8 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; /* * Different BTF ids get different trampolines, but they can still @@ -1651,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1666,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; } From 60bbe76a3f9ea4760f53a09cd2f00e211f7db9e3 Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Fri, 16 Oct 2026 13:33:17 +0000 Subject: [PATCH 531/660] bpf: Allow ksock connect/send calls from wq The initial ksock implementation, commit 7ae4eb14c5f9 ("bpf: Add ksock kfuncs"), kept LSM checks from the kfunc calls and disallowed asynchronous workqueues because worker credentials differ from those of the triggering task. However, discussions at Plumbers 2026[^1] revealed that bypassing LSM checks for BPF-created sockets could be acceptable. This patch allows workqueue use while retaining LSM checks for now, paving the way to call bpf_ksock_connect() and bpf_ksock_send() from non-sleepable programs, such as tracing programs, in future patches. Note that bpf_ksock_create() still checks: * for current to not be a kernel thread (thus excluding bpf workqueue callbacks) because the socket is created within a userspace task network namespace. It should, most of the time, be the process that triggered the SYSCALL program, but could be another task when called from bpf_task_work callbacks. * for current->nsproxy to be non-NULL since when called from a bpf_task_work callback and the task is in the do_exit() phase, nsproxy is torn down just before the remaining task work is executed. [^1]: https://lpc.events/event/20/contributions/2430/ Signed-off-by: Mahe Tardy Signed-off-by: Alexei Starovoitov Tested-by: Alan Maguire (Oracle) Link: https://patch.msgid.link/20261016133318.33999-1-mahe.tardy@gmail.com --- net/core/bpf_ksock.c | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/net/core/bpf_ksock.c b/net/core/bpf_ksock.c index e9943aeeadd3..7bf796193ab6 100644 --- a/net/core/bpf_ksock.c +++ b/net/core/bpf_ksock.c @@ -37,15 +37,6 @@ static void ksock_release_work_fn(struct work_struct *work) kfree(ks); } -static bool bpf_ksock_has_user_task_context(void) -{ - /* - * Task work can run from do_exit() after exit_nsproxy_namespaces() - * cleared current->nsproxy, while current is still not a kthread. - */ - return !(current->flags & PF_KTHREAD) && current->nsproxy; -} - __bpf_kfunc_start_defs(); /** @@ -59,7 +50,7 @@ __bpf_kfunc_start_defs(); * This function may sleep (sock_create), so it can only be used * in sleepable BPF programs (SYSCALL). * It cannot be called from a BPF workqueue callback because that callback - * does not retain the invoking task's namespace or security context. + * does not retain the invoking task's namespace. * * @opts: Pointer to struct bpf_ksock_create_opts with socket parameters. * @opts__sz: Size of the opts struct. @@ -74,11 +65,10 @@ bpf_ksock_create(const struct bpf_ksock_create_opts *opts, u32 opts__sz, int err; /* - * sock_create() derives the network namespace, credentials, and cgroup - * from current. Kernel threads, including BPF workqueue callbacks, do - * not carry the context of the task that invoked the BPF program. + * sock_create() derives the network namespace from current and task + * work can run from do_exit() after exit_nsproxy_namespaces() */ - if (!bpf_ksock_has_user_task_context()) { + if ((current->flags & PF_KTHREAD) || !current->nsproxy) { err = -EOPNOTSUPP; goto err_out; } @@ -165,9 +155,6 @@ __bpf_kfunc int bpf_ksock_connect(struct bpf_ksock *ks, struct sockaddr_storage sa; int addrlen; - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - if (!addr || addr__sz != sizeof(*addr)) return -EINVAL; @@ -247,9 +234,6 @@ __bpf_kfunc int bpf_ksock_send(struct bpf_ksock *ks, const void *data, }; int ret; - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - ret = kernel_sendmsg(ks->sock, &msg, &iov, 1, data__sz); return ret; From 242d8416f60cef4864d1a9de0943bd3f299e960c Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Fri, 16 Oct 2026 13:33:18 +0000 Subject: [PATCH 532/660] selftests/bpf: Test ksock connect/send calls from wq Extend test_ksock_wq() added in 'c93cbdb13f99 ("selftests/bpf: Add ksock test for async callback guard")' to test that the bpf_ksock_connect() and bpf_ksock_send() kfuncs are now allowed and work when called from workqueues. Signed-off-by: Mahe Tardy Signed-off-by: Alexei Starovoitov Tested-by: Alan Maguire (Oracle) Link: https://patch.msgid.link/20261016133318.33999-2-mahe.tardy@gmail.com --- .../selftests/bpf/prog_tests/ksock_wq.c | 29 ++++-- tools/testing/selftests/bpf/progs/ksock_wq.c | 88 ++++++++++++++++--- 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c index d6dc20b8f95b..12c2275e8177 100644 --- a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -4,6 +4,7 @@ #include #include "test_progs.h" +#include "network_helpers.h" #include "ksock_wq.skel.h" #define CALLBACK_WAIT_RETRIES 1000 @@ -13,13 +14,22 @@ void test_ksock_wq(void) { LIBBPF_OPTS(bpf_test_run_opts, opts); struct ksock_wq *skel; + char recv_data[sizeof(skel->data->send_data)]; u32 callback_done; - int err, i; + int err, i, port, rfd; skel = ksock_wq__open_and_load(); if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) return; + rfd = start_server(AF_INET, SOCK_DGRAM, "127.0.0.1", 0, 1000); + if (!ASSERT_OK_FD(rfd, "receiver socket")) + goto out; + port = get_socket_local_port(rfd); + if (!ASSERT_GE(port, 0, "receiver port")) + goto out; + skel->bss->remote_port = port; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), &opts); if (!ASSERT_OK(err, "run ksock_wq_start")) @@ -28,18 +38,27 @@ void test_ksock_wq(void) goto out; for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { - if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE) == 2) break; usleep(CALLBACK_WAIT_US); } callback_done = __atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE); - if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + if (!ASSERT_EQ(callback_done, 2, "workqueue callbacks completed")) goto out; - ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, - "workqueue create rejected"); + ASSERT_EQ(skel->bss->create_ret, -EOPNOTSUPP, "workqueue create rejected"); + + if (!ASSERT_EQ(skel->bss->connect_send_ret, sizeof(recv_data), + "workqueue send bytes")) + goto out; + err = recv(rfd, recv_data, sizeof(recv_data), 0); + if (ASSERT_EQ(err, sizeof(recv_data), "recv len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); out: + if (rfd >= 0) + close(rfd); ksock_wq__destroy(skel); } diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c index 16a1873d132e..fab356252d53 100644 --- a/tools/testing/selftests/bpf/progs/ksock_wq.c +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Isovalent */ #include "vmlinux.h" +#include #include #include "bpf_experimental.h" #include "bpf_tracing_net.h" @@ -14,15 +15,19 @@ struct ksock_wq_value { struct { __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, 1); + __uint(max_entries, 2); __type(key, u32); __type(value, struct ksock_wq_value); } work_map SEC(".maps"); -int create_err; +char send_data[] = "hello from bpf workqueue"; + +__be16 remote_port; +int connect_send_ret; +int create_ret; u32 callback_done; -static int ksock_wq_callback(void *map, int *key, void *value) +static int ksock_wq_create_cb(void *map, int *key, void *value) { struct bpf_ksock_create_opts opts = { .family = AF_INET, @@ -35,28 +40,85 @@ static int ksock_wq_callback(void *map, int *key, void *value) ks = bpf_ksock_create(&opts, sizeof(opts), &err); if (ks) bpf_ksock_release(ks); - create_err = err; + create_ret = err; __sync_fetch_and_add(&callback_done, 1); return 0; } +static int ksock_wq_connect_send_cb(void *map, int *key, void *value) +{ + union bpf_ksock_addr addr = { + .sin = { + .sin_family = AF_INET, + .sin_port = remote_port, + .sin_addr.s_addr = bpf_htonl(0x7f000001), + }, + }; + struct bpf_ksock *ks; + int err = -ENOENT; + + ks = ksock_ctx_get(); + if (!ks) + goto out; + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (!err) + err = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); +out: + connect_send_ret = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +static int wq_set_callback_start(int (*callback_fn)(void *, int *, void *), u32 key) +{ + struct ksock_wq_value *value; + int err; + + value = bpf_map_lookup_elem(&work_map, &key); + if (!value) + return -ENOENT; + + err = bpf_wq_init(&value->work, &work_map, 0); + if (err) + return err; + err = bpf_wq_set_callback(&value->work, callback_fn, 0); + if (err) + return err; + err = bpf_wq_start(&value->work, 0); + if (err) + return err; + + return 0; +} + SEC("syscall") int ksock_wq_start(void *ctx) { - struct ksock_wq_value *value; - u32 key = 0; - int err; + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; - value = bpf_map_lookup_elem(&work_map, &key); - if (!value) - return -ENOENT; - err = bpf_wq_init(&value->work, &work_map, 0); + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (!ks) + return err; + err = ksock_ctx_insert(ks); if (err) return err; - err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + + err = wq_set_callback_start(ksock_wq_connect_send_cb, 0); if (err) return err; - return bpf_wq_start(&value->work, 0); + + err = wq_set_callback_start(ksock_wq_create_cb, 1); + if (err) + return err; + + return 0; } char __license[] SEC("license") = "GPL"; From ca201ad6e92a1924138e83ff966f7a091e3bbe6e Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Thu, 15 Oct 2026 11:40:44 -0400 Subject: [PATCH 533/660] ARC: Strip libgcc debug information when generating BTF ARC links against the compiler's libgcc archive. The ARC GNU 2023.03 toolchain ships this library with debug information, including complex floating-point types in integer arithmetic helpers such as __udivdi3. These types reach vmlinux even though the kernel does not use complex floating-point arithmetic. When floating-point BTF encoding is enabled, pahole rejects those types: Complex, interval and imaginary float types are not supported Error while encoding BTF. FAILED: load BTF from .tmp_vmlinux1.BTF.1: No such file or directory KernelCI's upgrade to pahole 1.31 exposed this long-standing problem as failed ARC builds. As of 1.32, pahole still rejects these types, and older versions will stay in use, so the kernel build has to keep them out of vmlinux. Commit f122668ddcce ("ARC: Add eBPF JIT support") documented the same failure along with a workaround of disabling floating-point BTF encoding, but that also discards supported floating-point types from kernel BTF. For CONFIG_DEBUG_INFO_BTF, generate a debug-stripped copy of the selected libgcc archive in the output tree and link that copy. Preserve the archive structure so that only needed helpers are linked, and use the normal Kbuild objcopy rule to track input and command changes. The trade-off is that vmlinux in BTF-enabled builds no longer carries DWARF for the libgcc helpers. Debug information for kernel code, the installed toolchain and non-BTF builds stay unchanged. Reproduced and tested with ARC GCC 12.2.1 and pahole 1.31 and 1.32 using an ARCv2 tinyconfig with BPF_SYSCALL and DWARF4/BTF enabled. Before and after vmlinux.o files are identical after stripping debug information. Fixes: db16c1fe92d7 ("bpf: Generate BTF_KIND_FLOAT when linking vmlinux") Signed-off-by: Sasha Levin Signed-off-by: Alexei Starovoitov Acked-by: Alan Maguire (Oracle) Link: https://github.com/kernelci/kernelci-core/commit/e71b8c9f42253180596e734c18965a5aad80c8c5 Link: https://patch.msgid.link/20261015154044.4058618-1-sashal@kernel.org --- arch/arc/Makefile | 9 ++++++++- arch/arc/lib/Makefile | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/arc/Makefile b/arch/arc/Makefile index 868805ffcfea..4ea705a331c7 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -86,7 +86,14 @@ core-$(CONFIG_ARC_PLAT_TB10X) += arch/arc/plat-tb10x/ core-$(CONFIG_ARC_PLAT_AXS10X) += arch/arc/plat-axs10x/ core-$(CONFIG_ARC_SOC_HSDK) += arch/arc/plat-hsdk/ -libs-y += arch/arc/lib/ $(LIBGCC) +libs-y += arch/arc/lib/ + +ifdef CONFIG_DEBUG_INFO_BTF +export LIBGCC := $(LIBGCC) +libs-y += arch/arc/lib/libgcc.a +else +libs-y += $(LIBGCC) +endif boot := arch/arc/boot diff --git a/arch/arc/lib/Makefile b/arch/arc/lib/Makefile index 30158ae69fd4..2807ee24f27e 100644 --- a/arch/arc/lib/Makefile +++ b/arch/arc/lib/Makefile @@ -13,3 +13,11 @@ lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs-unaligned.o else lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs.o endif + +# libgcc can contain complex floating-point debug types unsupported by BTF. +# Strip only the debug information from a local copy of the selected archive. +always-$(CONFIG_DEBUG_INFO_BTF) += libgcc.a +OBJCOPYFLAGS_libgcc.a := --strip-debug + +$(obj)/libgcc.a: $(LIBGCC) FORCE + $(call if_changed,objcopy) From 4d8c247f1a21431a5cfb8c0237d5a35b7de4cc81 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 16 Oct 2026 12:18:20 +0200 Subject: [PATCH 534/660] bpf, s390: Mask the count of 32-bit shifts by register RFC9669 defines the shifts as dst <<= (src & mask), dst >>= (src & mask) and sign extending dst >>= (src & mask), where "shift operations use a mask of 0x3F (63) for 64-bit operations and 0x1F (31) for 32-bit operations" (Section 4.1), which the interpreter implements. On x86-64 and arm64 the hardware does so by itself, so both JITs emit the shift alone and conform. The 32-bit shifts by register of the s390x JIT do not conform to the spec. They emit sll, srl and sra with the count in the address of the second operand, of which the "rightmost six bits indicate the number of bit positions to be shifted" (z/Architecture Principles of Operation, SA22-7832-13), so a count of 32 to 63 clears the register, or fills it with the sign for sra, instead of shifting by the count masked with 31. The 64-bit sllg, srlg and srag take six bits as well, which is the mask there, so only the 32-bit forms differ. Put the count masked with 31 into %w1 and shift by that. A single risbg does the copy and the mask: it inserts bits 59 to 63 of the source into the same positions of %w1 and zeroes the rest, available since the general-instructions-extension facility of z10, below the z196 the JIT requires. One instruction more per shift by register, shifts by immediate are not affected since the verifier rejects a count at or beyond the width. # KERNEL_CMDLINE_EXTRA=nokaslr PLATFORM=s390x CROSS_COMPILE=s390x-linux-gnu- ./vmtest.sh -- \ ./test_progs -t jit_insns/shift_counts,jit_insns/shifts,jit_insns/alu_edges,jit_insns/calls [...] #191/8 jit_insns/alu_edges:OK #191/9 jit_insns/shifts:OK #191/10 jit_insns/shift_counts:OK #191/21 jit_insns/calls:OK #191/22 jit_insns/helper_calls:OK #191 jit_insns:OK #192/3 jit_insns_blinded/alu_edges:OK #192/5 jit_insns_blinded/shifts:OK #192 jit_insns_blinded:OK Summary: 2/7 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016101822.632891-1-daniel@iogearbox.net --- arch/s390/net/bpf_jit_comp.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 1b2566012b63..14c905697352 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -859,6 +859,14 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static void emit_shift_count32(struct bpf_jit *jit, u32 src_reg) +{ + /* risbg %w1,%src,59,128+63,0 */ + _EMIT6(0xec000000 | reg(REG_W1, src_reg) << 16 | 59 << 8 | (0x80 | 63), 0x0055); + REG_SET_SEEN(REG_W1); + REG_SET_SEEN(src_reg); +} + static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) { struct bpf_jit_probe probe; @@ -1431,9 +1439,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_LSH */ - case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ - /* sll %dst,0(%src) */ - EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* sll %dst,0(%w1) */ + EMIT4_DISP(0x89000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ @@ -1456,9 +1465,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_RSH */ - case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ - /* srl %dst,0(%src) */ - EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* srl %dst,0(%w1) */ + EMIT4_DISP(0x88000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ @@ -1481,9 +1491,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ARSH */ - case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ - /* sra %dst,%dst,0(%src) */ - EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= (u32) src & 31 */ + emit_shift_count32(jit, src_reg); + /* sra %dst,0(%w1) */ + EMIT4_DISP(0x8a000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ From b5fc954c39d419c3117c1a41a3ae4b2135986f7b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 16 Oct 2026 12:18:21 +0200 Subject: [PATCH 535/660] selftests/bpf: Restore the je <-> jmp distance in convergence check The bpf_jit_convergence je <-> jmp test reproduces the cycle fixed by commit c8831bdbfbab ("bpf, x64: Fix a jit convergence issue") with 125 bytes of x86-64 code between the je to l0 and l0 when the last goto l9 is two bytes and 128 when it is five, so that the je fits into an imm8 only while the jmp does not and the two flip each other's size from pass to pass. The recent JIT changes shortened that code by five bytes: the three loads and stores with a zero offset lost their displacement byte with commit 44df4d88aa63 ("bpf, x86: Omit the displacement byte for zero offsets"), and r2 += 0x1 became an inc and r3 &= 0x1 an and without REX.W with commit d59f47e031ae ("bpf, x86: Use shorter and faster forms for some ALU immediates"). At 120 and 123 bytes the je fits either way, nothing flips, and the test passes without exercising the fix. Add a five byte mov to the region to get the distance back. The je is forced into its long form again as the fix intends: # ./vmtest.sh -- ./test_progs -v -t verifier_jit_convergence [...] 96: 0f 84 7d 00 00 00 je L14 117: eb 83 jmp L6 119: bf 03 00 00 00 L14: movl $0x3, %edi [...] #668/1 verifier_jit_convergence/bpf_jit_convergence je <-> jmp:OK #668 verifier_jit_convergence:OK Summary: 1/1 PASSED, 0 SKIPPED, 0/0 FAILED Reported-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016101822.632891-2-daniel@iogearbox.net --- .../testing/selftests/bpf/progs/verifier_jit_convergence.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c index 9f3f2b7db450..4fd717334858 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c @@ -68,6 +68,13 @@ __naked void btf_jit_convergence_je_jmp(void) "r3 &= 0x1;" "r2 *= 0xa8;" "if r3 == 0x0 goto l2_%=;" + /* + * The x86-64 code from the je to l0 up to l0 has to be 125 bytes with + * the last goto l9 at two bytes and 128 with it at five, so that the + * je fits into an imm8 only while the jmp does not and the two keep + * flipping each other's size. The mov pads the region to that. + */ + "r3 = 0x1234;" "r1 += r2;" "r1 -= r7;" "r1 += 0x8;" From b4e14f47371c0b33b9d5bcfc9b286320a1a5bfcf Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 16 Oct 2026 12:18:22 +0200 Subject: [PATCH 536/660] selftests/bpf: Follow-up to JIT instruction tests Close gaps in the JIT instruction tests found by earlier Sashiko review, by comparing the programs with what reaches the JIT, and by a sweep through RFC9669 to construct tests for corner cases. Non-exhaustive list: - The staggered jumps loaded their operands once, so the verifier decided the later guards of ja and gotol and the later jumps of jeq_k and jlt32_k, and a landing short of the target passed with r0 at zero. Reload the operands per region and let r0 carry the region number from the jump to its target. - The matching 32-bit cmpxchg compared an r0 the mismatching one had zero extended. Reload r0 with the full value first and check that the source register keeps its upper half, as test_bpf did. - The 32-bit atomics wrote and read back only their word, so a JIT touching all 64 bits passed when the other half was zero. Fill and read back the whole slot. - The sub-word loads of the zero offset, offsets and probe_mem tests landed in a register holding nothing in particular. Set it to all ones first. - Let probe_mem load every size, sign extending too, and run ld_imm64 into every register, not only r1. - Drop the load-acquire probe: a JIT without the instruction fails like for any other one instead of skipping. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t jit_insns [...] #191/1 jit_insns/arena access with zero offset:OK #191/2 jit_insns/cast to a user address:OK #191/3 jit_insns/alu_imm:OK #191/4 jit_insns/alu_reg:OK #191/5 jit_insns/alu_imm_regs:OK #191/6 jit_insns/div_const:OK #191/7 jit_insns/alu_reg_pairs:OK #191/8 jit_insns/alu_edges:OK #191/9 jit_insns/shifts:OK #191/10 jit_insns/shift_counts:OK #191/11 jit_insns/atomics:OK #191/12 jit_insns/atomic_reg_pairs:OK #191/13 jit_insns/xadd_chain:OK #191/14 jit_insns/ld_imm64:OK #191/15 jit_insns/jmp_imm:OK #191/16 jit_insns/jmp_reg:OK #191/17 jit_insns/jmp_reg_pairs:OK #191/18 jit_insns/jmp_edges:OK #191/19 jit_insns/staggered_jumps:OK #191/20 jit_insns/max_jmp:OK #191/21 jit_insns/convergence:OK #191/22 jit_insns/calls:OK #191/23 jit_insns/helper_calls:OK #191/24 jit_insns/ldst_zero_off:OK #191/25 jit_insns/ldst_offsets:OK #191/26 jit_insns/acq_rel:OK #191/27 jit_insns/movsx:OK #191/28 jit_insns/jset_imm:OK #191/29 jit_insns/cmp_imm:OK #191/30 jit_insns/alu_consts:OK #191/31 jit_insns/bswap:OK #191/32 jit_insns/probe_mem:OK #191 jit_insns:OK #192/1 jit_insns_blinded/alu_imm:OK #192/2 jit_insns_blinded/alu_imm_regs:OK #192/3 jit_insns_blinded/alu_edges:OK #192/4 jit_insns_blinded/div_const:OK #192/5 jit_insns_blinded/shifts:OK #192/6 jit_insns_blinded/xadd_chain:OK #192/7 jit_insns_blinded/ld_imm64:OK #192/8 jit_insns_blinded/jmp_imm:OK #192/9 jit_insns_blinded/jmp_edges:OK #192/10 jit_insns_blinded/ldst_zero_off:OK #192/11 jit_insns_blinded/ldst_offsets:OK #192/12 jit_insns_blinded/jset_imm:OK #192/13 jit_insns_blinded/cmp_imm:OK #192/14 jit_insns_blinded/alu_consts:OK #192 jit_insns_blinded:OK Summary: 2/46 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016101822.632891-3-daniel@iogearbox.net --- .../selftests/bpf/prog_tests/jit_insns.c | 1104 ++++++++++++++--- 1 file changed, 913 insertions(+), 191 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c index dabf8a4abdfe..dabefdb8a353 100644 --- a/tools/testing/selftests/bpf/prog_tests/jit_insns.c +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -16,18 +16,18 @@ struct sweep_ctx { __u64 in[NR_CASES][2]; - __u64 out[NR_CASES][3]; + __u64 out[NR_CASES][4]; }; #define IN_OFF(k, j) (offsetof(struct sweep_ctx, in) + ((k) * 2 + (j)) * 8) -#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 3 + (j)) * 8) +#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 4 + (j)) * 8) /* The cases of one program: operands in, results out, expected results */ struct batch { struct bpf_insn *insns; int cnt, cap; struct sweep_ctx ctx; - __u64 exp[NR_CASES][3]; + __u64 exp[NR_CASES][4]; __u8 nr_out[NR_CASES]; int nr; char desc[80]; @@ -149,6 +149,65 @@ static int scratch_reg(int r1, int r2) return BPF_REG_9; } +static int size_bits(int size) +{ + switch (size) { + case BPF_B: + return 8; + case BPF_H: + return 16; + case BPF_W: + return 32; + default: + return 64; + } +} + +/* The first size bytes of v in memory, as a load of that size reads them */ +static __u64 head_bytes(__u64 v, int size) +{ + int bits = size_bits(size); + + if (bits == 64) + return v; +#if __BYTE_ORDER == __BIG_ENDIAN + return v >> (64 - bits); +#else + return v & ((1ULL << bits) - 1); +#endif +} + +/* v with its first size bytes in memory replaced by those of w */ +static __u64 head_merge(__u64 v, __u64 w, int size) +{ + int bits = size_bits(size); + __u64 mask; + + if (bits == 64) + return w; + mask = (1ULL << bits) - 1; +#if __BYTE_ORDER == __BIG_ENDIAN + return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits)); +#else + return (v & ~mask) | (w & mask); +#endif +} + +/* v as a sign extending load of that size reads it */ +static __s64 sign_extend(__u64 v, int size) +{ + switch (size) { + case BPF_B: + return (__s8)v; + case BPF_H: + return (__s16)v; + case BPF_W: + return (__s32)v; + default: + return v; + } +} + static const char *alu_name(int op) { switch (BPF_OP(op)) { @@ -189,12 +248,13 @@ static const char *jmp_name(int op) } } -/* ALU result computation, false when the operands are not valid */ +/* ALU result computation, with the ISA result for every operand */ enum { F_ALU32 = 1, F_SIGNED = 2 }; -static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) +static void alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) { bool is_signed = flags & F_SIGNED; + int mask = flags & F_ALU32 ? 31 : 63; /* Narrow operands for ALU32 */ if (flags & F_ALU32) { @@ -225,12 +285,15 @@ static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) *res = v1 ^ v2; break; case BPF_LSH: - *res = v1 << v2; + *res = v1 << (v2 & mask); break; case BPF_RSH: - *res = v1 >> v2; + *res = v1 >> (v2 & mask); break; case BPF_ARSH: + if (flags & F_ALU32) + v1 = (__u64)(__s32)v1; + v2 &= mask; *res = v1 >> v2; if (v2 > 0 && v1 > INT64_MAX) *res |= ~0ULL << (64 - v2); @@ -244,30 +307,28 @@ static bool alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) case BPF_MUL: *res = v1 * v2; break; + /* By zero: the quotient is zero and the dividend keeps, by -1: negation and zero */ case BPF_DIV: if (v2 == 0) - return false; - if (!is_signed) { + *res = 0; + else if (!is_signed) *res = v1 / v2; - } else { - if ((__s64)v2 == -1) /* Handled by the verifier */ - return false; + else if ((__s64)v2 == -1) + *res = -v1; + else *res = (__u64)((__s64)v1 / (__s64)v2); - } break; case BPF_MOD: if (v2 == 0) - return false; - if (!is_signed) { + *res = v1; + else if (!is_signed) *res = v1 % v2; - } else { - if ((__s64)v2 == -1) - return false; + else if ((__s64)v2 == -1) + *res = 0; + else *res = (__u64)((__s64)v1 % (__s64)v2); - } break; } - return true; } /* @@ -311,6 +372,27 @@ static void fill_pattern(struct batch *b, void *arg, int dbits, int sbits, batch_flush(b); } +/* Operands at the edges: zero, one and the limits of every width */ +static const __u64 edges[] = { + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000, + 0xffffffff, 0x100000000ULL, 0x7fffffffffffffffULL, 0x8000000000000000ULL, + 0xffffffff80000000ULL, 0xffffffff7fffffffULL, 0xffffffffffffffffULL, +}; + +static void fill_edges(struct batch *b, void *arg, bool imm, emit_fn emit) +{ + int i, j; + + for (i = 0; i < ARRAY_SIZE(edges); i++) + for (j = 0; j < ARRAY_SIZE(edges); j++) { + /* The immediate has to fit its 32 bits */ + if (imm && (__s64)edges[j] != (__s32)edges[j]) + continue; + emit(b, arg, edges[i], edges[j]); + } + batch_flush(b); +} + /* ALU operations with an immediate and with a register for all magnitudes */ static const int alu_ops[] = { BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, @@ -328,9 +410,16 @@ static void emit_alu_imm(struct batch *b, void *arg, __s64 dst, __s64 imm) __u64 res; int k; - if (!alu_result(&res, dst, (__s32)imm, a->op, a->flags)) + /* The verifier rejects a zero divisor and a shift count beyond the width */ + if ((a->op == BPF_DIV || a->op == BPF_MOD) && !imm) return; - k = batch_case(b, BPF_REG_3, dst, 0); + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (__u64)imm >= (alu32 ? 32 : 64)) + return; + if (a->op == BPF_NEG) + imm = 0; + alu_result(&res, dst, (__s32)imm, a->op, a->flags); + k = batch_case(b, BPF_REG_3, dst, (__s32)imm); LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_K, BPF_REG_1, 0, a->flags & F_SIGNED ? 1 : 0, imm)); @@ -345,8 +434,11 @@ static void emit_alu_reg(struct batch *b, void *arg, __s64 dst, __s64 src) __u64 res; int k; - if (!alu_result(&res, dst, src, a->op, a->flags)) + /* Shift counts beyond the width have their own subtest */ + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (alu32 ? (__u32)src : (__u64)src) >= (alu32 ? 32 : 64)) return; + alu_result(&res, dst, src, a->op, a->flags); k = batch_case(b, BPF_REG_3, dst, src); LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); @@ -398,6 +490,41 @@ static void test_alu_reg(void) test_alu_magnitudes(true); } +static void test_alu_edges(void) +{ + static const struct alu_arg ops[] = { + { BPF_MOV }, { BPF_AND }, { BPF_OR }, { BPF_XOR }, { BPF_LSH }, { BPF_RSH }, + { BPF_ARSH }, { BPF_ADD }, { BPF_SUB }, { BPF_MUL }, { BPF_DIV }, { BPF_MOD }, + { BPF_DIV, F_SIGNED }, { BPF_MOD, F_SIGNED }, { BPF_NEG }, + }; + struct batch b; + int o, f, r, i; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + struct alu_arg a = { ops[o].op, ops[o].flags | (f ? F_ALU32 : 0) }; + + for (r = 0; r < (a.op == BPF_NEG ? 1 : 2); r++) { + snprintf(b.desc, sizeof(b.desc), "alu%d_%s%s_%c edges", + f ? 32 : 64, a.flags & F_SIGNED ? "s" : "", alu_name(a.op), + r ? 'x' : 'k'); + if (a.op == BPF_NEG) { + for (i = 0; i < ARRAY_SIZE(edges); i++) + emit_alu_imm(&b, &a, edges[i], 0); + batch_flush(&b); + } else if (r) { + fill_edges(&b, &a, false, emit_alu_reg); + } else { + fill_edges(&b, &a, true, emit_alu_imm); + } + } + } + } + batch_fini(&b); +} + /* ALU operations with an immediate for every destination register */ static const int alu_reg_ops[] = { BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_LSH, BPF_RSH, BPF_ARSH, @@ -516,11 +643,7 @@ static __u64 div_result(int op, __u32 flags, __s32 imm, __u64 n) { __u64 res; - /* LLONG_MIN / -1 == LLONG_MIN, LLONG_MIN % -1 == 0, same for INT_MIN */ - if ((flags & F_SIGNED) && imm == -1) - res = op == BPF_MOD ? 0 : -n; - else - alu_result(&res, n, (__s32)imm, op, flags); + alu_result(&res, n, (__s32)imm, op, flags); return flags & F_ALU32 ? (__u32)res : res; } @@ -570,37 +693,46 @@ static void test_div_const(void) /* ALU operations for all combinations of operand registers */ static void alu_reg_pairs(struct batch *b, int op, __u32 flags) { - bool alu32 = flags & F_ALU32; - __u64 dst, src, res, same; - int rd, rs, rc, k; + bool alu32 = flags & F_ALU32, divmod = op == BPF_DIV || op == BPF_MOD; + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X; + int off = flags & F_SIGNED ? 1 : 0, rd, rs, rc, k, n, nr = 1; + __u64 dst, srcs[3], src, res, same; if (flags & F_SIGNED) dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; else dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; - src = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL; + srcs[0] = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL; if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - src &= 31; - alu_result(&res, dst, src, op, flags); - alu_result(&same, src, src, op, flags); - if (alu32) { - res = (__u32)res; - same = (__u32)same; + srcs[0] &= 31; + /* A zero divisor, in 32 bits also one with only the upper half set */ + if (divmod) { + srcs[nr++] = 0; + if (alu32) + srcs[nr++] = 1ULL << 32; } snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_x register pairs", alu32 ? 32 : 64, flags & F_SIGNED ? "s" : "", alu_name(op)); - for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { - for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { - rc = scratch_reg(rd, rs); - k = batch_case(b, rc, rd == rs ? src : dst, src); - LD_IN(b, k, 0, rc, rd); - if (rs != rd) - LD_IN(b, k, 1, rc, rs); - EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X, - rd, rs, flags & F_SIGNED ? 1 : 0, 0)); - ST_OUT(b, k, 0, rc, rd); - batch_expect(b, k, 0, rd == rs ? same : res); + for (n = 0; n < nr; n++) { + src = srcs[n]; + alu_result(&res, dst, src, op, flags); + alu_result(&same, src, src, op, flags); + if (alu32) { + res = (__u32)res; + same = (__u32)same; + } + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, rd == rs ? src : dst, src); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_RAW_INSN(code, rd, rs, off, 0)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, rd == rs ? same : res); + } } } batch_flush(b); @@ -622,16 +754,17 @@ static void test_alu_reg_pairs(void) batch_fini(&b); } +static const __s64 shift_vals[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dword < 0, word > 0 */ + 0xfedcba0198765432LL, /* dword < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ +}; +static const int shift_ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH }; + /* Shifts for all shift counts, by immediate, by register and by the shifted register */ static void test_shifts(void) { - static const __s64 vals[] = { - 0x0123456789abcdefLL, /* dword > 0, word < 0 */ - 0xfedcba9876543210LL, /* dword < 0, word > 0 */ - 0xfedcba0198765432LL, /* dword < 0, word < 0 */ - 0x0123458967abcdefLL, /* dword > 0, word > 0 */ - }; - static const int ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH }; struct batch b; int o, f, v, imm, k; @@ -640,14 +773,14 @@ static void test_shifts(void) for (f = 0; f < 2; f++) { int bits = f ? 32 : 64; - for (o = 0; o < ARRAY_SIZE(ops); o++) { - int op = ops[o]; + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; snprintf(b.desc, sizeof(b.desc), "alu%d_%s all shift counts", bits, alu_name(op)); - for (v = 0; v < ARRAY_SIZE(vals); v++) { + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { for (imm = 0; imm < bits; imm++) { - __u64 reg = vals[v], res; + __u64 reg = shift_vals[v], res; if (f) reg = op == BPF_ARSH ? (__u64)(__s32)reg : @@ -656,7 +789,7 @@ static void test_shifts(void) if (f) res = (__u32)res; - k = batch_case(&b, BPF_REG_3, vals[v], imm); + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); if (f) EMIT(&b, BPF_ALU32_IMM(op, BPF_REG_1, imm)); @@ -665,7 +798,7 @@ static void test_shifts(void) ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); batch_expect(&b, k, 0, res); - k = batch_case(&b, BPF_REG_3, vals[v], imm); + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); if (f) @@ -696,6 +829,54 @@ static void test_shifts(void) batch_fini(&b); } +/* + * Shifts by register with counts at and above the width and negative ones, + * which the ISA masks by the width. Apart in a subtest of its own, as not + * every JIT does so. + */ +static void test_shift_counts(void) +{ + static const __s64 counts[] = { 32, 33, 63, 64, 65, 127, -1, -3, 1LL << 32 | 3 }; + struct batch b; + int o, f, v, c, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_x counts beyond the width", + bits, alu_name(op)); + for (c = 0; c < ARRAY_SIZE(counts); c++) { + __u64 cnt = f ? (__u32)counts[c] : counts[c]; + + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { + __u64 reg = shift_vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, cnt & (bits - 1), op, 0); + k = batch_case(&b, BPF_REG_3, shift_vals[v], counts[c]); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + } + batch_flush(&b); + } + } + batch_fini(&b); +} + /* Atomic operations on the stack for all magnitudes of memory and source operand */ static const int atomic_ops[] = { BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, @@ -723,16 +904,18 @@ static void emit_atomic(struct batch *b, void *arg, __s64 dst, __s64 src) fetch = a->op & BPF_FETCH ? (w ? (__u32)dst : dst) : src; k = batch_case(b, BPF_REG_3, dst, src); + /* The kept value fills the slot, a 32-bit operation leaves the other half alone */ + emit_ld_imm64(b, BPF_REG_0, keep); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); EMIT(b, BPF_STX_MEM(a->size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); - emit_ld_imm64(b, BPF_REG_0, keep); EMIT(b, BPF_ATOMIC_OP(a->size, a->op, BPF_REG_10, BPF_REG_2, MEM_SLOT)); - EMIT(b, BPF_LDX_MEM(a->size, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_2); ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); - batch_expect(b, k, 0, w ? (__u32)res : res); + batch_expect(b, k, 0, head_merge(keep, w ? (__u32)res : res, a->size)); batch_expect(b, k, 1, fetch); batch_expect(b, k, 2, keep); } @@ -746,21 +929,30 @@ static void emit_cmpxchg(struct batch *b, void *arg, __s64 dst, __s64 src) k = batch_case(b, BPF_REG_3, dst, src); LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + /* The inverted value fills the slot, a 32-bit operation leaves the other half alone */ + EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1)); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); EMIT(b, BPF_STX_MEM(size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); /* Mismatch: the memory keeps its value, r0 gets it */ - EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1)); - EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_0); - /* Match: the memory gets the source, r0 the old value */ + /* + * Match: the memory gets the source, r0 the old value. r0 holds the + * full value, so the 32-bit compare has to ignore its upper half, and + * the source register has to keep its own. + */ + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_0); EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); - EMIT(b, BPF_LDX_MEM(size, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_1); ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + ST_OUT(b, k, 3, BPF_REG_3, BPF_REG_2); batch_expect(b, k, 0, old); - batch_expect(b, k, 1, w ? (__u32)src : src); + batch_expect(b, k, 1, head_merge(~(__u64)dst, w ? (__u32)src : src, size)); batch_expect(b, k, 2, old); + batch_expect(b, k, 3, src); } static void test_atomics(void) @@ -797,7 +989,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op) int rd, rs, rc, k, n; if (w) { - mem = 0x01234567U; + mem = 0xfedcba9801234567ULL; upd = 0x76543210U; } else { mem = 0x0123456789abcdefULL; @@ -809,18 +1001,19 @@ static void atomic_reg_pairs(struct batch *b, int size, int op) op & BPF_FETCH ? "_fetch" : ""); for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { - __u64 regs[10], old = mem, new, src, cmp; + __u64 regs[10], old = w ? (__u32)mem : mem, new, src, cmp; /* The source would be the pointer, r0 the compare value */ if (rs == rd || (op == BPF_CMPXCHG && rd == BPF_REG_0)) continue; rc = scratch_reg(rd, rs); k = batch_case(b, rc, mem, upd); - /* Memory from in0, r0 inverted, source from in1, pointer */ - LD_IN(b, k, 0, rc, rs); - EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT)); + /* r0 inverted into the slot, memory from in0, source from in1, pointer */ LD_IN(b, k, 0, rc, BPF_REG_0); EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + LD_IN(b, k, 0, rc, rs); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT)); LD_IN(b, k, 1, rc, rs); EMIT(b, BPF_MOV64_REG(rd, BPF_REG_10)); EMIT(b, BPF_ATOMIC_OP(size, op, rd, rs, MEM_SLOT)); @@ -829,7 +1022,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op) if (rd != BPF_REG_0) ST_OUT(b, k, n++, rc, BPF_REG_0); ST_OUT(b, k, n++, rc, rs); - EMIT(b, BPF_LDX_MEM(size, rs, BPF_REG_10, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, MEM_SLOT)); ST_OUT(b, k, n, rc, rs); /* Model with the register aliasing of r0 */ @@ -854,7 +1047,7 @@ static void atomic_reg_pairs(struct batch *b, int size, int op) if (rd != BPF_REG_0) batch_expect(b, k, n++, regs[BPF_REG_0]); batch_expect(b, k, n++, regs[rs]); - batch_expect(b, k, n, new); + batch_expect(b, k, n, head_merge(~mem, new, size)); } } batch_flush(b); @@ -905,13 +1098,14 @@ static void test_xadd_chain(void) free(insns); } -/* 64-bit immediate loads for all magnitudes and byte patterns */ -static void ld_imm64_case(struct batch *b, __u64 imm) +/* 64-bit immediate loads for all magnitudes, byte patterns and registers */ +static void ld_imm64_case(struct batch *b, int rd, __u64 imm) { - int k = batch_case(b, BPF_REG_3, imm, 0); + int rc = scratch_reg(rd, rd); + int k = batch_case(b, rc, imm, 0); - emit_ld_imm64(b, BPF_REG_1, imm); - ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + emit_ld_imm64(b, rd, imm); + ST_OUT(b, k, 0, rc, rd); batch_expect(b, k, 0, imm); } @@ -925,7 +1119,13 @@ static void test_ld_imm64(void) { 1, 0x81, 0, 0xff }, /* positive and zero */ { 0x80, 0x80, 0, 0xff }, /* negative and zero */ }; - int block = 64, bit, adj, sign, p, pattern, index; + /* Zero, a 32-bit, a sign extended 32-bit and a full immediate for every register */ + static const __u64 imms[] = { + 0, 1, 0x7fffffff, 0x80000000, 0xffffffff, 0x100000000ULL, + 0xffffffffffffffffULL, 0xffffffff80000000ULL, 0xffffffff7fffffffULL, + 0x0123456789abcdefULL, + }; + int block = 64, bit, adj, sign, p, pattern, index, rd, i; struct batch b; if (!batch_init(&b)) @@ -934,7 +1134,7 @@ static void test_ld_imm64(void) for (bit = 0; bit <= 62; bit++) for (adj = -block / 2; adj < block / 2; adj++) for (sign = -1; sign <= 1; sign += 2) - ld_imm64_case(&b, sign * ((1LL << bit) + adj)); + ld_imm64_case(&b, BPF_REG_1, sign * ((1LL << bit) + adj)); batch_flush(&b); for (p = 0; p < ARRAY_SIZE(bytes); p++) { @@ -956,10 +1156,15 @@ static void test_ld_imm64(void) imm = (imm << 8) | (__u8)byte; } rand = rand * 1664525 + 1013904223; - ld_imm64_case(&b, imm); + ld_imm64_case(&b, BPF_REG_1, imm); } batch_flush(&b); } + snprintf(b.desc, sizeof(b.desc), "ld_imm64 registers"); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (i = 0; i < ARRAY_SIZE(imms); i++) + ld_imm64_case(&b, rd, imms[i]); + batch_flush(&b); batch_fini(&b); } @@ -1066,6 +1271,27 @@ static void test_jmp_reg(void) test_jmp_magnitudes(true); } +static void test_jmp_edges(void) +{ + struct batch b; + int o, f, r; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + for (r = 0; r < 2; r++) { + struct jmp_arg a = { jmp_ops[o], f, r }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c edges", f ? "32" : "", + jmp_name(a.op), r ? 'x' : 'k'); + fill_edges(&b, &a, !r, emit_jmp); + } + } + } + batch_fini(&b); +} + /* A conditional jump on two registers, with the operands swapped for n == 1 */ static void jmp_reg_pair(struct batch *b, bool is32, int op, int rd, int rs, int n) { @@ -1136,10 +1362,14 @@ static struct bpf_insn jmp_insn(bool ja32) /* * Taken jumps over regions of doubling length, forward or backward, up to * the maximal offset of the jump: 16 bits, or 2^17 for the 32-bit - * unconditional jump. The not taken path of a conditional jump and the - * guard of an unconditional one lead to an exit with the region number, so - * the verifier sees every instruction reachable and a jump landing anywhere - * else than on its target fails the run. + * unconditional jump. The operands are reloaded in every region, so that + * the verifier cannot decide the jumps from the bounds of the region + * before and fold the regions away. r0 gains the region number before the + * jump and loses it again only on the target, so the not taken path of a + * conditional jump, the guard of an unconditional one and a jump landing + * short of its target, or forward past it, exit with a nonzero r0. A + * backward jump overshooting its target comes back to itself and hangs + * the run instead. */ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool backward, const char *desc) @@ -1152,18 +1382,20 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool .ctx_size_in = sizeof(ctx), ); int max = ja32 ? 1 << 17 : INT16_MAX; - int len = 128 + 2 * max, i = 0, n = 0, gap, fd; + int len = 256 + 2 * max, i = 0, n = 0, gap, fd; struct bpf_insn *insns; insns = malloc(len * sizeof(*insns)); if (!ASSERT_OK_PTR(insns, "insns")) return; - insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8); - insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); for (gap = backward ? 4 : 2; gap <= max; gap = gap < max / 2 ? gap * 2 : max) { int k, a, t, x, g, j; + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, 8); + insns[i++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, ++n); if (backward) { /* * Over the region to its guard and jump, which comes back @@ -1172,7 +1404,8 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool */ a = i++; t = i; - for (k = 0; k < gap - 3; k++) + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); + for (k = 0; k < gap - 3 - ja; k++) insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); x = i++; g = i; @@ -1181,7 +1414,6 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool j = i; insns[i] = *jmp; set_jmp_off(&insns[i++], ja32, t - (j + 1)); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n); insns[i++] = BPF_EXIT_INSN(); insns[a] = jmp_insn(ja32); set_jmp_off(&insns[a], ja32, g - (a + 1)); @@ -1193,10 +1425,10 @@ static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); insns[i] = *jmp; set_jmp_off(&insns[i++], ja32, gap); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, ++n); - for (k = 0; k < gap - 2; k++) + for (k = 0; k < gap - 1; k++) insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); insns[i++] = BPF_EXIT_INSN(); + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); } if (gap == max) break; @@ -1334,50 +1566,6 @@ static void prog_run(int fd, __u64 in0, __u64 in1, __u64 expected, const char *d ASSERT_EQ(ctx.res, expected, buf); } -static int size_bits(int size) -{ - switch (size) { - case BPF_B: - return 8; - case BPF_H: - return 16; - case BPF_W: - return 32; - default: - return 64; - } -} - -/* The first size bytes of v in memory, as a load of that size reads them */ -static __u64 head_bytes(__u64 v, int size) -{ - int bits = size_bits(size); - - if (bits == 64) - return v; -#if __BYTE_ORDER == __BIG_ENDIAN - return v >> (64 - bits); -#else - return v & ((1ULL << bits) - 1); -#endif -} - -/* v with its first size bytes in memory replaced by those of w */ -static __u64 head_merge(__u64 v, __u64 w, int size) -{ - int bits = size_bits(size); - __u64 mask; - - if (bits == 64) - return w; - mask = (1ULL << bits) - 1; -#if __BYTE_ORDER == __BIG_ENDIAN - return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits)); -#else - return (v & ~mask) | (w & mask); -#endif -} - static void test_ldst_zero_off(void) { static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; @@ -1398,8 +1586,9 @@ static void test_ldst_zero_off(void) for (k = 0; k < NR_PAIRS; k++) { int rd = pair_reg(b, k); - /* Load with a zero offset, also into the base */ + /* Load with a zero offset over all ones, also into the base */ prog_start(&p); + EMIT(&p, BPF_MOV64_IMM(rd, -1)); EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); EMIT(&p, BPF_LDX_MEM(size, rd, rb, 0)); prog_end(&p, rd); @@ -1431,8 +1620,7 @@ static void test_ldst_zero_off(void) for (j = 0; j < ARRAY_SIZE(inputs); j++) { __s64 v = head_bytes(inputs[j], size); - v = size == BPF_B ? (__s8)v : - size == BPF_H ? (__s16)v : (__s32)v; + v = sign_extend(v, size); prog_run(fd, inputs[j], 0, v, desc); } close(fd); @@ -1614,6 +1802,12 @@ static int other_reg(int r1, int r2) return BPF_REG_9; } +static void off_prog_ld_imm64(struct off_prog *p, int reg, __u64 imm) +{ + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + /* Look up the value of key into r0, exit with 1 when the lookup fails */ static void off_prog_lookup(struct off_prog *p, int map_fd, int key) { @@ -1700,8 +1894,11 @@ static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd, off_prog_start(&p, map_fd, rb); for (i = 0; i < nr; i++) { + /* The base gets reloaded when it is the destination, all ones otherwise */ if (rd == rb) EMIT(&p, BPF_LDX_MEM(BPF_DW, rb, BPF_REG_10, SLOT_BASE)); + else + EMIT(&p, BPF_MOV64_IMM(rd, -1)); if (kind == K_LDX) EMIT(&p, BPF_LDX_MEM(size, rd, rb, off[i])); else @@ -1722,7 +1919,7 @@ static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd, __u64 v = load_bytes(m->mem + MEM_BASE + off[i], size); if (kind == K_LDSX) - v = size == BPF_B ? (__s8)v : size == BPF_H ? (__s16)v : (__s32)v; + v = sign_extend(v, size); if (res[i] != v) { snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d off=%d", kind == K_LDX ? "ldx" : "ldsx", size, rb, rd, off[i]); @@ -1873,7 +2070,7 @@ static void test_movsx(void) } /* Branch on dst and imm, with 1 in res when taken and 0 otherwise */ -static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm) +static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm, int off) { int rd = pat_regs[d], rt = pair_reg(d, 1); @@ -1881,9 +2078,9 @@ static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm) EMIT(p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); EMIT(p, BPF_MOV64_IMM(rt, 1)); if (is64) - EMIT(p, BPF_JMP_IMM(op, rd, imm, 1)); + EMIT(p, BPF_JMP_IMM(op, rd, imm, off)); else - EMIT(p, BPF_JMP32_IMM(op, rd, imm, 1)); + EMIT(p, BPF_JMP32_IMM(op, rd, imm, off)); EMIT(p, BPF_MOV64_IMM(rt, 0)); prog_end(p, rt); } @@ -1903,7 +2100,7 @@ static void test_jset_imm(void) for (is64 = false; ; is64 = true) { for (m = 0; m < ARRAY_SIZE(masks); m++) { - jmp_prog(&p, is64, BPF_JSET, d, masks[m]); + jmp_prog(&p, is64, BPF_JSET, d, masks[m], 1); snprintf(desc, sizeof(desc), "jset%d dst=r%d mask=0x%x", is64 ? 64 : 32, rd, masks[m]); fd = prog_load(&p, desc); @@ -1975,7 +2172,7 @@ static void test_cmp_imm(void) for (is64 = false; ; is64 = true) { for (o = 0; o < ARRAY_SIZE(ops); o++) { for (m = 0; m < ARRAY_SIZE(imms); m++) { - jmp_prog(&p, is64, ops[o], d, imms[m]); + jmp_prog(&p, is64, ops[o], d, imms[m], 1); snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d imm=%d", is64 ? 64 : 32, ops[o], rd, imms[m]); fd = prog_load(&p, desc); @@ -1990,6 +2187,18 @@ static void test_cmp_imm(void) prog_run(fd, inputs[j], 0, taken, desc); } close(fd); + if (m) + continue; + /* A jump over nothing lands on the same path either way */ + jmp_prog(&p, is64, ops[o], d, imms[m], 0); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d off=0", + is64 ? 64 : 32, ops[o], rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, 0, desc); + close(fd); } } if (is64) @@ -2122,7 +2331,15 @@ static void test_bswap(void) static void test_probe_mem(void) { - int kfunc_id, type_id, b, k, j, fd; + static const struct { + int size; + bool sx; + } loads[] = { + { BPF_B, false }, { BPF_H, false }, { BPF_W, false }, { BPF_DW, false }, + { BPF_B, true }, { BPF_H, true }, { BPF_W, true }, + }; + int kfunc_id, type_id, b, k, l, j, fd; + struct bpf_insn call; struct btf *btf; struct prog p; char desc[64]; @@ -2136,30 +2353,46 @@ static void test_probe_mem(void) if (!ASSERT_GT(kfunc_id, 0, "bpf_rdonly_cast id") || !ASSERT_GT(type_id, 0, "timespec64 id")) return; + call = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, kfunc_id); - /* The kernel copy of the ctx through an untrusted pointer in every register */ + /* + * The kernel copy of the ctx through an untrusted pointer in every + * register, in every size, zero and sign extending over all ones + */ for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { int rb = pat_regs[b]; for (k = 0; k < NR_PAIRS; k++) { int rd = pair_reg(b, k); - prog_start(&p); - EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6)); - EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); - EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, - 0, kfunc_id)); - if (rb != BPF_REG_0) - EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0)); - EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, rb, 0)); - prog_end(&p, rd); - snprintf(desc, sizeof(desc), "probe_mem base=r%d dst=r%d", rb, rd); - fd = prog_load(&p, desc); - if (fd < 0) - return; - for (j = 0; j < ARRAY_SIZE(inputs); j++) - prog_run(fd, inputs[j], 0, inputs[j], desc); - close(fd); + for (l = 0; l < ARRAY_SIZE(loads); l++) { + int size = loads[l].size, mode = loads[l].sx ? BPF_MEMSX : BPF_MEM; + + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, call); + if (rb != BPF_REG_0) + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0)); + if (rd != rb) + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | mode | size, rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "probe_mem%s size=%d base=r%d dst=r%d", + loads[l].sx ? "sx" : "", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + if (loads[l].sx) + v = sign_extend(v, size); + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } } } @@ -2171,8 +2404,7 @@ static void test_probe_mem(void) EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, 0, 0, (__u32)addr)); EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, addr >> 32)); EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); - EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, - 0, kfunc_id)); + EMIT(&p, call); EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0)); prog_end(&p, BPF_REG_3); snprintf(desc, sizeof(desc), "probe_mem addr=0x%llx", addr); @@ -2184,24 +2416,6 @@ static void test_probe_mem(void) } } -/* Whether the JIT supports load-acquire and store-release */ -static bool acq_rel_ok(void) -{ - LIBBPF_OPTS(bpf_prog_load_opts, lopts, .prog_flags = BPF_F_SLEEPABLE); - struct prog p; - int fd; - - prog_start(&p); - EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0)); - EMIT(&p, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)); - prog_end(&p, BPF_REG_0); - fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p.insns, p.cnt, &lopts); - if (fd < 0) - return false; - close(fd); - return true; -} - /* * Load-acquire and store-release of every size with every register as * base, on a stack slot holding in0: the loads zero extend like the plain @@ -2215,10 +2429,6 @@ static void test_acq_rel(void) char desc[64]; int b, k, s, j, fd; - if (!acq_rel_ok()) { - test__skip(); - return; - } for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { int rb = pat_regs[b]; @@ -2340,6 +2550,502 @@ static void test_max_jmp(void) max_jmp(true, false); } +/* + * Fillers of two and three bytes of native code on x86-64: a 32-bit and a + * 64-bit move between two low registers, which give a region every length + * from two bytes on and which constant blinding leaves alone. + */ +static int fill_bytes(struct bpf_insn *insns, int n, int bytes) +{ + if (bytes & 1) { + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + bytes -= 3; + } + for (; bytes > 0; bytes -= 2) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static void prog_result(struct bpf_insn *insns, int *n) +{ + insns[(*n)++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10); + insns[(*n)++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[(*n)++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[(*n)++] = BPF_EXIT_INSN(); +} + +/* A region of x86-64 bytes through fill_bytes(), or of so many 64 and 32-bit moves */ +struct fill { + int bytes, wide, narrow; +}; + +static int fill_moves(struct bpf_insn *insns, int n, int wide, int narrow) +{ + while (wide-- > 0) + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + while (narrow-- > 0) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static int fill_region(struct bpf_insn *insns, int n, const struct fill *f) +{ + if (f->wide || f->narrow) + return fill_moves(insns, n, f->wide, f->narrow); + return fill_bytes(insns, n, f->bytes); +} + +static int fill_insns(const struct fill *f) +{ + return f->wide + f->narrow + f->bytes / 2 + 1; +} + +/* + * Jumps whose lengths depend on each other: a conditional jump forward + * over a region, a jump back from its end to its start or to the jump in + * front of it, and an unconditional jump out of it. The region takes every + * length around the short branch limit, in front of the exit and behind + * it, so the three jumps cross the limit at different lengths and in both + * directions. A chain of nop jumps in it, gone one per pass, shifts the + * phase in which the cycle meets the padding passes, as settling jumps + * elsewhere do. This is generic to any JIT with short and long branch + * forms: the ranges of the running one are probed, see find_ranges(), and + * the cycle swept across each of them. r0 goes in with 0 to 3 and picks + * one of the four paths through the jumps. + */ +static void jump_cycle(bool back_cond, bool to_start, int inner, const struct fill *front, + const struct fill *back, const char *desc) +{ + struct bpf_insn *insns; + int n = 0, disp, head, chain, to_back[2], out, i, fd; + + insns = malloc((32 + 2 * inner + fill_insns(front) + fill_insns(back)) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, desc)) + return; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, see nop_chain() */ + disp = n; + n += inner; + head = n++; + n = fill_region(insns, n, front); + chain = n; + for (i = 0; i < inner; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = BPF_JMP_IMM(BPF_JA, 0, 0, inner - 1 - i); + } + to_back[0] = n++; + to_back[1] = n++; + out = n++; + insns[to_back[0]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, n - to_back[0] - 1); + insns[to_back[1]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 3, n - to_back[1] - 1); + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + n = fill_region(insns, n, back); + /* Taken with 2 and not with 4, so the verifier keeps the condition */ + if (back_cond) + insns[n] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 4, head + to_start - n - 1); + else + insns[n] = BPF_JMP_IMM(BPF_JA, 0, 0, head + to_start - n - 1); + n++; + insns[head] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, n - head - 1); + insns[out] = BPF_JMP_IMM(BPF_JA, 0, 0, n - out - 1); + prog_result(insns, &n); + + fd = load_prog(insns, n, desc); + free(insns); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 12, desc); + prog_run(fd, 2, 0, 12, desc); + prog_run(fd, 3, 0, 14, desc); + close(fd); +} + +/* + * A region which ends in a chain of jumps to its end: the last one jumps + * to the next instruction, which the JIT drops as a nop, whereupon the one + * before it becomes one, so the chain goes away one jump per pass and the + * jump over the region shrinks late, in the passes which pad the image. + * The filler in front of the chain puts that jump around the short branch + * limit, as a conditional and as an unconditional one, over a chain of ja + * and one of gotol. Each jump of the chain is the target of a conditional + * jump in front of the region on a value the verifier cannot see, as the + * chain is dead behind its first jump otherwise. + */ +static void nop_chain(bool over_cond, bool ja32, int len, int fill) +{ + struct bpf_insn insns[256]; + int n = 0, disp, over, chain, i, fd; + char desc[64]; + + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, filled in once the chain is placed */ + disp = n; + n += len; + if (!over_cond) + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, 1); + over = n++; + n = fill_bytes(insns, n, fill); + chain = n; + for (i = 0; i < len; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, len - 1 - i) : + BPF_JMP_IMM(BPF_JA, 0, 0, len - 1 - i); + } + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + if (over_cond) + insns[over] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 2, n - over - 1); + else + insns[over] = BPF_JMP_IMM(BPF_JA, 0, 0, n - over - 1); + prog_result(insns, &n); + + snprintf(desc, sizeof(desc), "nop chain of %s under %s len=%d fill=%d", + ja32 ? "gotol" : "ja", over_cond ? "jcc" : "ja", len, fill); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 11, desc); + prog_run(fd, 2, 0, 13, desc); + prog_run(fd, 3, 0, 13, desc); + close(fd); +} + +/* The native length of a program's image, 0 without a JIT */ +static __u32 jited_len(int fd) +{ + struct bpf_prog_info info = {}; + __u32 len = sizeof(info); + + if (!ASSERT_OK(bpf_prog_get_info_by_fd(fd, &info, &len), "prog info")) + return 0; + return info.jited_prog_len; +} + +/* A conditional jump over the moves, for the length of its image */ +static int probe_len(int wide, int narrow, __u32 *len) +{ + struct bpf_insn *insns; + int n = 0, fd; + + insns = malloc((wide + narrow + 8) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "probe insns")) + return -1; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, wide + narrow); + n = fill_moves(insns, n, wide, narrow); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + fd = load_prog(insns, n, "branch range probe"); + free(insns); + if (fd < 0) + return -1; + *len = jited_len(fd); + close(fd); + return 0; +} + +/* + * The short branch ranges of the running JIT, as counts of 64-bit moves a + * conditional jump reaches over, and the native lengths of the two moves. + * The image of the probe grows by the moves alone until the jump does not + * reach over them any more and takes a longer form, which adds the + * difference of the forms: find the first count that does so in every + * power of two of moves up to the 16-bit offset of the jump, as a JIT may + * have several forms. Without a JIT or with fixed length branches there is + * none. + */ +static int find_ranges(int *ranges, int max, int *wide_bytes, int *narrow_bytes) +{ + __u32 base, len; + int excess, plateau = 0, c, lo, hi, mid, k = 0; + + if (probe_len(8, 0, &base) || !base) + return 0; + if (probe_len(24, 0, &len)) + return 0; + *wide_bytes = (len - base) / 16; + if (probe_len(8, 16, &len)) + return 0; + *narrow_bytes = (len - base) / 16; + if (!*wide_bytes || !*narrow_bytes) + return 0; + for (c = 32; k < max && c <= 16384; c *= 2) { + if (probe_len(c, 0, &len)) + return k; + excess = len - base - (c - 8) * *wide_bytes; + if (excess <= plateau) + continue; + lo = c / 2; + hi = c; + while (hi - lo > 1) { + mid = (lo + hi) / 2; + if (probe_len(mid, 0, &len)) + return k; + if (len - base - (mid - 8) * *wide_bytes > plateau) + hi = mid; + else + lo = mid; + } + ranges[k++] = hi; + plateau = excess; + } + return k; +} + +static int gcd(int a, int b) +{ + while (b) { + int t = a % b; + + a = b; + b = t; + } + return a; +} + +/* The jump cycle with the region in front of the exit and then behind it */ +static void cycle_bytes(int kind, int inner, int bytes) +{ + struct fill front = { .bytes = bytes }, back = {}; + char desc[96]; + + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d front=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); + front.bytes = 0; + back.bytes = bytes; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d back=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); +} + +/* The jump cycle with a region of so many native bytes of moves */ +static void cycle_moves(int kind, int range, int bytes, int wide, int narrow) +{ + struct fill front = {}, back = {}; + char desc[96]; + int b; + + for (b = 0; b * narrow <= bytes; b++) + if ((bytes - b * narrow) % wide == 0) + break; + if (b * narrow > bytes) + return; + front.wide = (bytes - b * narrow) / wide; + front.narrow = b; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s range %d at %d bytes", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", range, bytes); + jump_cycle(kind & 1, kind & 2, 0, &front, &back, desc); +} + +static void test_convergence(void) +{ + /* The chain gone, the jump over it is at and below the limit */ + static const int fills[] = { 100, 110, 116, 120 }; + /* The last jump of a chain is a nop the verifier already removes */ + static const int inners[] = { 0, 2, 3, 4, 5 }; + int ranges[4], wide, narrow, g, nr, bytes, lo, hi, c, f, i, l, k; + + /* The x86-64 imm8 range, byte by byte */ + for (c = 0; c < 4; c++) + for (i = 0; i < ARRAY_SIZE(inners); i++) + for (f = 100; f <= 132; f++) + cycle_bytes(c, inners[i], f); + for (c = 0; c < 4; c++) + for (f = 0; f < ARRAY_SIZE(fills); f++) + for (l = 2; l <= 26; l++) + nop_chain(c & 1, c & 2, l, fills[f]); + + /* + * The short branch ranges of this JIT, which the fillers of 64 and + * 32-bit moves approach at the granularity their lengths allow + */ + nr = find_ranges(ranges, ARRAY_SIZE(ranges), &wide, &narrow); + for (k = 0; k < nr; k++) { + printf("short branch range at %d bytes, moves of %d and %d bytes\n", + ranges[k] * wide, wide, narrow); + g = gcd(wide, narrow); + lo = ranges[k] * wide - 128; + hi = ranges[k] * wide + 32; + for (c = 0; c < 4; c++) + for (bytes = lo; bytes <= hi; bytes += g) + cycle_moves(c, ranges[k] * wide, bytes, wide, narrow); + } +} + +/* + * Calls into a subprogram with all five arguments, which clobbers a subset + * of r6-r9 and uses the deepest stack slot the budget leaves, directly or + * through another one clobbering the rest, so the JIT has to save and + * restore what it uses. + */ +/* Values to keep in r6-r9 across calls */ +static const __u64 kept[] = { + 0x1111111111111111ULL, 0x2222222222222222ULL, + 0x3333333333333333ULL, 0x4444444444444444ULL, +}; + +static void test_calls(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + __u64 expected = 1 + 2 + 3 + 4 + 5; + struct bpf_insn insns[64]; + struct pat_ctx ctx; + char desc[64]; + int nested, mask, n, call, r, fd, leaf; + + /* The frames above the leaf take up to 32 bytes each of the budget */ + leaf = (is_large_stack_supported() ? 2048 : 512) - 2 * 32; + for (r = 0; r < 4; r++) + expected += kept[r]; + opts.ctx_in = &ctx; + opts.ctx_size_in = sizeof(ctx); + for (nested = 0; nested < 2; nested++) { + for (mask = 0; mask < 16; mask++) { + n = 0; + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -8); + for (r = 0; r < 4; r++) { + insns[n++] = BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_6 + r, + 0, 0, (__u32)kept[r]); + insns[n++] = BPF_RAW_INSN(0, 0, 0, 0, kept[r] >> 32); + } + for (r = 1; r <= 5; r++) + insns[n++] = BPF_MOV64_IMM(r, r); + call = n++; + for (r = 6; r <= 9; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -8); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + if (nested) { + /* Clobber the subset, pass the arguments on */ + for (r = 0; r < 4; r++) + if (mask & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + call = n++; + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + } + /* Clobber the rest, sum the arguments through the deepest slot */ + for (r = 0; r < 4; r++) + if ((nested ? ~mask : mask) & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -leaf); + insns[n++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); + for (r = 3; r <= 5; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -leaf); + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); + insns[n++] = BPF_EXIT_INSN(); + + snprintf(desc, sizeof(desc), "call%s clobbering 0x%x", + nested ? " nested" : "", mask); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + memset(&ctx, 0, sizeof(ctx)); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(ctx.res, expected, desc); + close(fd); + } + } +} + +/* + * Helper calls with up to five arguments and a kfunc call: a dynptr over a + * map value, a read from it into the stack and its size, with r6-r9 live + * across the calls. The returns, the bytes read, the size and r6-r9 add up. + */ +static void test_helper_calls(void) +{ + static const int offs[] = { 0, 8, 1000, 4088 }; + struct off_prog p; + struct btf *btf; + __u8 mem[4096]; + __u64 expected; + __u32 key = 0; + char desc[64]; + int map_fd, kfunc_id, i, r, fd; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_dynptr_size", BTF_KIND_FUNC); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_dynptr_size id")) + return; + for (i = 0; i < sizeof(mem); i++) + mem[i] = i * 7 + 3; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "helper_mem", sizeof(key), sizeof(mem), 1, + NULL); + if (!ASSERT_GE(map_fd, 0, "helper_mem map")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, mem, 0), "helper_mem update")) + goto out; + + for (i = 0; i < ARRAY_SIZE(offs); i++) { + p.cnt = 0; + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -64)); + for (r = 0; r < 4; r++) + off_prog_ld_imm64(&p, BPF_REG_6 + r, kept[r]); + off_prog_lookup(&p, map_fd, key); + /* The dynptr at -80 over the value, the returns summed at -96 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_0)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, sizeof(mem))); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_3, 0)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_4, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -80)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_from_mem)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* Eight bytes from the offset into -88 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -88)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, 8)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_3, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -80)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_4, offs[i])); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_5, 0)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_read)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* The size through the kfunc */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -80)); + EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, + kfunc_id)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -88)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + for (r = BPF_REG_6; r <= BPF_REG_9; r++) + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -64)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES)); + snprintf(desc, sizeof(desc), "helper calls off=%d", offs[i]); + fd = off_prog_load(&p, desc, false); + if (fd < 0) + goto out; + expected = sizeof(mem) + load_bytes(mem + offs[i], BPF_DW); + for (r = 0; r < 4; r++) + expected += kept[r]; + prog_run(fd, 0, 0, expected, desc); + close(fd); + } +out: + close(map_fd); +} + void test_jit_insns(void) { RUN_TESTS(jit_insns_arena); @@ -2353,8 +3059,12 @@ void test_jit_insns(void) test_div_const(); if (test__start_subtest("alu_reg_pairs")) test_alu_reg_pairs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); if (test__start_subtest("shifts")) test_shifts(); + if (test__start_subtest("shift_counts")) + test_shift_counts(); if (test__start_subtest("atomics")) test_atomics(); if (test__start_subtest("atomic_reg_pairs")) @@ -2369,10 +3079,18 @@ void test_jit_insns(void) test_jmp_reg(); if (test__start_subtest("jmp_reg_pairs")) test_jmp_reg_pairs(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); if (test__start_subtest("staggered_jumps")) test_staggered_jumps(); if (test__start_subtest("max_jmp")) test_max_jmp(); + if (test__start_subtest("convergence")) + test_convergence(); + if (test__start_subtest("calls")) + test_calls(); + if (test__start_subtest("helper_calls")) + test_helper_calls(); if (test__start_subtest("ldst_zero_off")) test_ldst_zero_off(); if (test__start_subtest("ldst_offsets")) @@ -2409,6 +3127,8 @@ void serial_test_jit_insns_blinded(void) test_alu_imm(); if (test__start_subtest("alu_imm_regs")) test_alu_imm_regs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); if (test__start_subtest("div_const")) test_div_const(); if (test__start_subtest("shifts")) @@ -2419,6 +3139,8 @@ void serial_test_jit_insns_blinded(void) test_ld_imm64(); if (test__start_subtest("jmp_imm")) test_jmp_imm(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); if (test__start_subtest("ldst_zero_off")) test_ldst_zero_off(); if (test__start_subtest("ldst_offsets")) From d2141481a764efdfaac70965ea28e8925f6a6b80 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 16 Oct 2026 17:55:59 +0200 Subject: [PATCH 537/660] bpf: Free the subprog jump tables once the CFG passes are done The jump tables which compute_subprog_jts() builds are only read through bpf_insn_successors() once CFG discovery is done. Its last caller is bpf_compute_scev(), which runs after bpf_compute_live_registers(), so free the tables right after it instead of keeping them until the verifier environment is torn down. With large insn_array maps, this also drops them before do_check() starts to allocate states. Dead code removal runs after do_check(), so a removed subprogram no longer owns a table and adjust_subprog_starts_after_remove() can go back to not caring about the table pointers. Suggested-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016155559.652461-1-daniel@iogearbox.net --- kernel/bpf/fixups.c | 9 +-------- kernel/bpf/verifier.c | 2 ++ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index b2fa7e8bedf3..608621d7f225 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -477,11 +477,7 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, if (j > i) { struct bpf_prog_aux *aux = env->prog->aux; - int move, k; - - /* The removed subprogs own their jump tables. */ - for (k = i; k < j; k++) - kvfree(env->subprog_info[k].jt); + int move; /* move fake 'exit' subprog as well */ move = env->subprog_cnt + 1 - j; @@ -489,9 +485,6 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, memmove(env->subprog_info + i, env->subprog_info + j, sizeof(*env->subprog_info) * move); - /* The vacated tail still holds copies of the moved entries. */ - memset(env->subprog_info + i + move, 0, - sizeof(*env->subprog_info) * (j - i)); env->subprog_cnt -= j - i; /* remove func_info and its aux */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ec136cf377c1..62f8951126ba 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23487,6 +23487,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + bpf_free_subprog_jts(env); + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; From 2d1f101ee619435873ed8a23fcb3bd09204a5024 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 9 Oct 2026 10:24:27 +0800 Subject: [PATCH 538/660] bpf: Claim the per-CPU send_signal irq_work before filling it irq_work_is_busy() cannot see the per-CPU send_signal_work while it is being filled: the check only matches after irq_work_queue() has claimed the work. A concurrent caller therefore passes it, both callers race for the same irq_work, and the loser's signal is silently lost along with its task reference while the queued work runs with a mix of both callers' fields. kprobe, tracepoint and perf_event programs exclude each other on the same CPU through bpf_prog_active, so for two callers to meet, one of the programs has to be a raw tracepoint or an fentry one. And since commit 87c544108b61 ("bpf: Send signals asynchronously if !preemptible") this path runs with IRQs enabled too, so a hard IRQ can interrupt the fill as well, not only an NMI. Found by code inspection and reproduced with a perf_event NMI program racing an fentry one. Claim the work through work->task itself: cmpxchg() it from NULL to the target task before touching the other fields, and set it back to NULL once the callback has consumed them. A context finding the work claimed returns the documented -EBUSY. Fixes: 1bc7896e9ef4 ("bpf: Fix deadlock with rq_lock in bpf_send_signal()") Signed-off-by: Yuan Chen Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261009022427.968304-1-chenyuan@kylinos.cn --- kernel/trace/bpf_trace.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 195f78db9bda..5465e7eba238 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -823,6 +823,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { struct send_signal_irq_work { struct irq_work irq_work; + /* The claim: NULL marks the slot free for the next caller. */ struct task_struct *task; u32 sig; enum pid_type type; @@ -842,6 +843,8 @@ static void do_bpf_send_signal(struct irq_work *entry) group_send_sig_info(work->sig, siginfo, work->task, work->type); put_task_struct(work->task); + /* Release once the fields are consumed. */ + smp_store_release(&work->task, NULL); } static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) @@ -885,14 +888,14 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc return -EINVAL; work = this_cpu_ptr(&send_signal_work); - if (irq_work_is_busy(&work->irq_work)) + if (cmpxchg(&work->task, NULL, task)) return -EBUSY; /* Add the current task, which is the target of sending signal, * to the irq_work. The current task may change when queued * irq works get executed. */ - work->task = get_task_struct(task); + get_task_struct(task); work->has_siginfo = siginfo == &info; if (work->has_siginfo) copy_siginfo(&work->info, &info); From d9efbaed7e614278002c4c1e476b62b3f20ed5df Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Fri, 16 Oct 2026 10:13:12 -0700 Subject: [PATCH 539/660] bpf, x86: Support fetching AND/OR/XOR atomics in arena x86-64 has no single instruction for a fetching AND/OR/XOR, so the JIT lowers them to a CMPXCHG loop. That loop could not be used against arena memory: it contains two memory accesses, the load of the old value and the CMPXCHG itself, either of which can fault when the arena page goes away, while the verifier reserves only one exception table entry per instruction. bpf_jit_supports_insn() therefore rejected the three opcodes and such programs failed to load with BPF_ATOMIC stores into R1 arena is not allowed Emit the same loop with R12-indexed addressing and give each of the two accesses its own exception table entry. The verifier still reserves one per instruction, so the JIT accounts for the extra entry itself by rescanning the instruction stream in bpf_int_jit_compile() before the extable is sized, as s390 does in bpf_jit_alloc() for its BPF_XCHG lowering. The rescan sits after the extra-pass early exit, so a program is counted exactly once. Drop the bpf_jit_supports_insn() rejection. A loop without the load would need only one entry, since a CMPXCHG that loses the comparison reloads RAX, but its first CMPXCHG would then almost always lose. Every operation would pay for a second locked CMPXCHG, which makes the fetching insn more expensive than the load and CMPXCHG loop programs write by hand today. The second entry is paid for once, in the JIT. Both entries resume past the whole loop rather than past the faulting instruction alone, with the fetch destination cleared, so a fault cannot re-enter the loop, and both are reported as writes since the BPF insn is a read-modify-write. This repurposes the INSN_LEN field of the fixup as a resume distance rather than an instruction length; a sequence long enough to overflow its 8 bits would first have to exceed BPF_MAX_INSN_SIZE, which do_jit() rejects before the image is used. The loop needs RAX for CMPXCHG and saves R0 in BPF_REG_AX, which it also substitutes for R0 when either operand is R0. ex_handler_bpf() may then have to name r10 as the register holding the arena address or the one to clear on fault; reg2pt_regs[] already has that entry. An access through a number, made by a program loaded with BPF_F_ARENA_SCALAR, keeps its address in BPF_REG_AX itself (see arena_scalar_access()), which is where the loop would save R0. Save R0 on the stack in that case, as the div and mod lowering does. An R0 operand stays in that stack slot: the new value is built from it, RAX ends up holding the old value that R0 is to return, and the slot is dropped instead of popped. Both entries resume before that pop or drop, so a fault leaves the stack balanced. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Acked-by: Tejun Heo Link: https://patch.msgid.link/20261016171319.1623447-2-puranjay@kernel.org --- arch/x86/net/bpf_jit_comp.c | 295 ++++++++++++++++++++++++++---------- 1 file changed, 217 insertions(+), 78 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 3975a94ac812..004e2ddeb4f6 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -239,6 +239,13 @@ static const int reg2pt_regs[] = { [BPF_REG_AX] = offsetof(struct pt_regs, r10), }; +static bool is_atomic_fetch_op(const struct bpf_insn *insn) +{ + return insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH); +} + /* * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 * which need extra byte of encoding. @@ -1784,6 +1791,93 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } +/* + * A fetching AND/OR/XOR can't be implemented with a single x86 insn, so do a + * CMPXCHG loop. @index_reg is X86_REG_R12 for an arena access or -1 otherwise. + * Unless @r0_on_stack, the caller has substituted BPF_REG_AX for R0 in + * @dst_reg and @src_reg. + * + * Both the load and the CMPXCHG can fault when accessing the arena, so their + * addresses are handed back in @fault[0] and @fault[1]. A fault has to resume + * at @resume, which is past the loop and past the move that delivers the old + * value, but before R0 is restored. Both are NULL for the non-arena case, + * which needs no fixup. + * + * R0 is expected to be saved in BPF_REG_AX by the caller, or pushed on the + * stack if @r0_on_stack. In that case an R0 operand stays in the stack slot, + * RAX ends up holding the old value that R0 is to return, and the slot is + * dropped rather than popped. + */ +static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + u32 src_reg, int index_reg, bool r0_on_stack, + u8 **fault, u8 **resume) +{ + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + bool operand_on_stack = r0_on_stack && src_reg == BPF_REG_0; + u8 *branch_target, *prog = *pprog; + int err; + + branch_target = prog; + + /* Load old value */ + if (fault) + fault[0] = prog; + if (index_reg < 0) + emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, insn->off); + else + emit_ldx_index(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, + index_reg, insn->off); + + /* + * Perform the (commutative) operation locally, put the result in + * the AUX_REG. + */ + if (operand_on_stack) { + /* mov r11, [rsp]: the operand is the R0 saved there, RAX is the old value */ + if (is64) + EMIT4(0x4C, 0x8B, 0x1C, 0x24); + else + EMIT4(0x44, 0x8B, 0x1C, 0x24); + } else { + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); + } + maybe_emit_mod(&prog, AUX_REG, src_reg, is64); + EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], + add_2reg(0xC0, AUX_REG, src_reg)); + + /* Attempt to swap in new value */ + if (fault) + fault[1] = prog; + if (index_reg < 0) + err = emit_atomic_rmw(&prog, BPF_CMPXCHG, dst_reg, AUX_REG, + insn->off, BPF_SIZE(insn->code)); + else + err = emit_atomic_rmw_index(&prog, BPF_CMPXCHG, BPF_SIZE(insn->code), + dst_reg, AUX_REG, index_reg, insn->off); + if (WARN_ON(err)) + return err; + + /* ZF tells us whether we won the race. If it's cleared we need to try again. */ + EMIT2(X86_JNE, -(prog - branch_target) - 2); + /* Return the pre-modification value, which RAX already is for R0 */ + if (!operand_on_stack) + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + + if (resume) + *resume = prog; + + /* Restore R0 after clobbering RAX */ + if (!r0_on_stack) + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); + else if (operand_on_stack) + EMIT2(0x41, 0x5B); /* pop r11, dropping the operand */ + else + EMIT1(0x58); /* pop rax */ + + *pprog = prog; + return 0; +} + /* * Metadata encoding for exception handling in JITed code. * @@ -1797,7 +1891,12 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | * +-----------+-------------+--------+-----------+---------+----------+ * - * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). + * - INSN_LEN (8 bits): How far past the faulting insn to resume. That is its own length + * for a single-insn access, but the distance to the end of the whole + * sequence where one BPF insn became several, as for the CMPXCHG loop + * of a fetching AND/OR/XOR. A sequence long enough to overflow this + * field would first have to exceed BPF_MAX_INSN_SIZE, which do_jit() + * rejects with -EFAULT before the image is used. * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the @@ -1844,7 +1943,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) bpf_prog_report_arena_violation(is_write, addr, regs->ip); } - /* jump over faulting load and clear dest register */ + /* jump over the faulting access, or the sequence it is part of, and clear dest register */ if (reg != DONT_CLEAR) *(unsigned long *)((void *)regs + reg) = 0; regs->ip += insn_len; @@ -1852,6 +1951,44 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) return true; } +/* + * Record an arena access that may fault. @fault_ip is the address of the + * faulting insn in the RO image, @resume_off how far past it execution has to + * resume: for a multi-insn lowering that is the end of the whole sequence, not + * the end of the one insn. + */ +static int emit_arena_exentry(struct bpf_prog *bpf_prog, u8 *image, u8 *rw_image, + int *excnt, u8 *fault_ip, u32 resume_off, + u32 fixup_reg, u32 arena_reg, bool is_write, s16 off) +{ + struct exception_table_entry *ex; + s64 delta; + + if (!bpf_prog->aux->extable) + return 0; + + if (*excnt >= bpf_prog->aux->num_exentries) { + pr_err("arena extable bug\n"); + return -EFAULT; + } + ex = &bpf_prog->aux->extable[(*excnt)++]; + + delta = fault_ip - (u8 *)&ex->insn; + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + + ex->insn = delta; + ex->data = EX_TYPE_BPF | FIELD_PREP(DATA_ARENA_OFFSET_MASK, off); + ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, resume_off) | + FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | + FIELD_PREP(FIXUP_REG_MASK, fixup_reg) | + FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; + + return 0; +} + static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, bool *regs_used) { @@ -2967,28 +3104,8 @@ div_done: } populate_extable: { - struct exception_table_entry *ex; - u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; bool is_write; - s64 delta; - - if (!bpf_prog->aux->extable) - break; - - if (excnt >= bpf_prog->aux->num_exentries) { - pr_err("mem32 extable bug\n"); - return -EFAULT; - } - ex = &bpf_prog->aux->extable[excnt++]; - - delta = _insn - (u8 *)&ex->insn; - /* switch ex to rw buffer for writes */ - ex = (void *)rw_image + ((void *)ex - (void *)image); - - ex->insn = delta; - - ex->data = EX_TYPE_BPF; /* * src_reg/dst_reg holds the address in the arena region with upper @@ -3024,14 +3141,12 @@ populate_extable: is_write = true; } - ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | - FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | - FIELD_PREP(FIXUP_REG_MASK, fixup_reg); - ex->fixup |= FIXUP_ARENA_ACCESS; - if (is_write) - ex->fixup |= FIXUP_ARENA_WRITE; - - ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); + err = emit_arena_exentry(bpf_prog, image, rw_image, &excnt, + image + proglen + (start_of_ldx - temp), + prog - start_of_ldx, fixup_reg, + arena_reg, is_write, insn->off); + if (err) + return err; } break; @@ -3177,20 +3292,12 @@ populate_extable: fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: case BPF_STX | BPF_ATOMIC | BPF_DW: { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; u32 real_src_reg = src_reg; u32 real_dst_reg = dst_reg; + bool is_atomic_fetch = is_atomic_fetch_op(insn); u8 *old_prog; - bool is_atomic_fetch = - (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)); - if (is_atomic_fetch) { - /* - * Can't be implemented with a single x86 insn. - * Need to do a CMPXCHG loop. - */ + if (is_atomic_fetch) { /* Will need RAX as a CMPXCHG operand so save R0 */ old_prog = prog; emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); @@ -3211,34 +3318,11 @@ populate_extable: } } if (is_atomic_fetch) { - u8 *branch_target = prog; - /* Load old value */ - emit_ldx(&prog, BPF_SIZE(insn->code), - BPF_REG_0, real_dst_reg, insn->off); - /* - * Perform the (commutative) operation locally, - * put the result in the AUX_REG. - */ - emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); - maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); - EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], - add_2reg(0xC0, AUX_REG, real_src_reg)); - /* Attempt to swap in new value */ - err = emit_atomic_rmw(&prog, BPF_CMPXCHG, - real_dst_reg, AUX_REG, - insn->off, - BPF_SIZE(insn->code)); - if (WARN_ON(err)) + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, -1, false, + NULL, NULL); + if (err) return err; - /* - * ZF tells us whether we won the race. If it's - * cleared we need to try again. - */ - EMIT2(X86_JNE, -(prog - branch_target) - 2); - /* Return the pre-modification value */ - emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); - /* Restore R0 after clobbering RAX */ - emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); break; } @@ -3262,6 +3346,53 @@ populate_extable: fallthrough; case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: + if (is_atomic_fetch_op(insn)) { + u32 real_src_reg = src_reg, real_dst_reg = dst_reg; + /* + * An access through a number has its address in + * BPF_REG_AX (see arena_scalar_access()), so R0 is + * saved on the stack instead, as div and mod do. + */ + bool r0_on_stack = dst_reg == BPF_REG_AX; + u8 *fault[2], *resume; + int j; + + /* Will need RAX as a CMPXCHG operand so save R0 */ + if (r0_on_stack) { + EMIT1(0x50); /* push rax */ + } else { + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + if (src_reg == BPF_REG_0) + real_src_reg = BPF_REG_AX; + if (dst_reg == BPF_REG_0) + real_dst_reg = BPF_REG_AX; + } + + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, X86_REG_R12, + r0_on_stack, fault, &resume); + if (err) + return err; + + /* + * The page may be unmapped between the load and the + * CMPXCHG, so both need an entry. Both report a write, + * since the BPF insn is a read-modify-write. + */ + for (j = 0; j < 2; j++) { + err = emit_arena_exentry(bpf_prog, image, rw_image, + &excnt, + image + proglen + + (fault[j] - temp), + resume - fault[j], + reg2pt_regs[real_src_reg], + reg2pt_regs[real_dst_reg], + true, insn->off); + if (err) + return err; + } + break; + } start_of_ldx = prog; if (bpf_atomic_is_load_store(insn)) @@ -4707,6 +4838,22 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr padding = true; goto skip_init_addrs; } + + /* + * An arena fetching AND/OR/XOR is lowered to a CMPXCHG loop whose load + * and CMPXCHG can both fault, one more entry than the verifier reserved. + * The extra pass of a program with subprograms returned above, so the + * count is adjusted once. + */ + for (i = 0; prog->aux->arena && i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + + if (BPF_CLASS(insn->code) == BPF_STX && + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && + is_atomic_fetch_op(insn)) + prog->aux->num_exentries++; + } + addrs = kvmalloc_objs(*addrs, prog->len + 1); if (!addrs) goto out_addrs; @@ -4761,7 +4908,9 @@ out_image: * The number of entries in extable is the number of BPF_LDX * insns that access kernel memory via "pointer to BTF type". * The verifier changed their opcode from LDX|MEM|size - * to LDX|PROBE_MEM|size to make JITing easier. + * to LDX|PROBE_MEM|size to make JITing easier. Arena + * accesses add one each, and an arena fetching AND/OR/XOR + * one more, see the adjustment of num_exentries above. */ u32 align = __alignof__(struct exception_table_entry); u32 extable_size = prog->aux->num_exentries * @@ -5035,16 +5184,6 @@ bool bpf_jit_supports_arena_scalar(void) bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { - if (!in_arena) - return true; - switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) - return false; - } return true; } From 9e35f216e5c5c834a81f1c7a410531271d7a987a Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Fri, 16 Oct 2026 10:13:13 -0700 Subject: [PATCH 540/660] selftests/bpf: Test fetching AND/OR/XOR atomics in arena The arena and/or/xor tests never exercised the fetching form of the instruction. Commit 2897b1e2a2f4 ("selftests/bpf: Fix arena_atomics failure due to llvm change") deliberately switched them to __c11_atomic_fetch_*() with memory_order_relaxed, which clang lowers to a non-fetching locked instruction when the result is unused, because x86-64 could not JIT the fetching one against an arena. It can now, so check the returned old value with __sync_fetch_and_*(). Keep a __c11_atomic_fetch_*() of each width per operation with the result discarded: that spelling is what selects the non-fetching insn, which would otherwise lose its only coverage, on every architecture rather than just x86. Compiling the prog with clang 22 before and after confirms the split: 6 non-fetching and no fetching arena and/or/xor before, 6 and 22 after. The new globals go in their operation's block, and the or block keeps its u32s first, so .addr_space.1 stays free of the padding holes that clang before 18 cannot fill when it is built without address space casts. Add a fetch_r0 test for the two register assignments the x86 JIT has to special-case, where the operand and where the arena pointer is R0 and BPF_REG_AX is substituted for it. Clang picks its own registers and will not reliably produce either, so spell the instructions out. The values have bits set above 32 so that a truncating move on that path shows up. Add both forms to the uaf test as well, since the substitution only reaches the exception table when such an access faults, and check that the fault cleared the fetched register and left the address alone. They fault on the page after the freed one: the freed page is the first of the arena, whose 32-bit address is 0, which would make the address check empty. The address is read back afterwards, which gives 0 if the JIT's exception table entry recovered the fault, but not if the arena had mapped its zeroed scratch page and let the access succeed. The powerpc64 JIT does not clear the register of a faulting atomic yet, so those checks are skipped there. Both spell addr_space_cast textually, which needs LLVM 19, so uaf gains the __BPF_FEATURE_ADDR_SPACE_CAST guard that every other textual use in the file already has. It was plain C before and built on older clang. Drop x86 from the exclusion list of the uaf test, which covers the fault path. Its page is gone before the first access, so it faults on the load; the CMPXCHG entry is only reached when the page goes away between the two accesses, which a test cannot arrange. Finally cover fetching AND/OR/XOR through a number in verifier_arena_scalar, on x86-64 and arm64. Both widths are run with the operand in a register and in R0, which is where the x86-64 JIT keeps the operand on the stack because BPF_REG_AX holds the address. The 32-bit operands have bits set above 32 so that a missing zero-extension shows. The page is also freed first, after storing a nonzero value in it so that an access which did not fault would show, and the test checks that the fetched register comes back 0 and R0 is left alone, and that reading the address back gives 0. R0 is compared with a value derived from the page address: the verifier would decide a comparison with a constant at load time and drop it. Signed-off-by: Puranjay Mohan Signed-off-by: Alexei Starovoitov Acked-by: Tejun Heo Link: https://patch.msgid.link/20261016171319.1623447-3-puranjay@kernel.org --- .../selftests/bpf/prog_tests/arena_atomics.c | 53 ++++ .../selftests/bpf/progs/arena_atomics.c | 211 ++++++++++++--- .../bpf/progs/verifier_arena_scalar.c | 256 ++++++++++++++++++ 3 files changed, 482 insertions(+), 38 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index 1ad5d03d07ad..222c63b15de0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -68,6 +68,12 @@ static void test_and(struct arena_atomics *skel) ASSERT_EQ(skel->arena->and64_value, 0x010ull << 32, "and64_value"); ASSERT_EQ(skel->arena->and32_value, 0x010, "and32_value"); + + ASSERT_EQ(skel->arena->and64_result, 0x110ull << 32, "and64_result"); + ASSERT_EQ(skel->arena->and32_result, 0x110, "and32_result"); + + ASSERT_EQ(skel->arena->and64_noreturn_value, 0x010ull << 32, "and64_noreturn_value"); + ASSERT_EQ(skel->arena->and32_noreturn_value, 0x010, "and32_noreturn_value"); } static void test_or(struct arena_atomics *skel) @@ -85,6 +91,12 @@ static void test_or(struct arena_atomics *skel) ASSERT_EQ(skel->arena->or64_value, 0x111ull << 32, "or64_value"); ASSERT_EQ(skel->arena->or32_value, 0x111, "or32_value"); + + ASSERT_EQ(skel->arena->or64_result, 0x110ull << 32, "or64_result"); + ASSERT_EQ(skel->arena->or32_result, 0x110, "or32_result"); + + ASSERT_EQ(skel->arena->or64_noreturn_value, 0x111ull << 32, "or64_noreturn_value"); + ASSERT_EQ(skel->arena->or32_noreturn_value, 0x111, "or32_noreturn_value"); } static void test_xor(struct arena_atomics *skel) @@ -102,6 +114,12 @@ static void test_xor(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xor64_value, 0x101ull << 32, "xor64_value"); ASSERT_EQ(skel->arena->xor32_value, 0x101, "xor32_value"); + + ASSERT_EQ(skel->arena->xor64_result, 0x110ull << 32, "xor64_result"); + ASSERT_EQ(skel->arena->xor32_result, 0x110, "xor32_result"); + + ASSERT_EQ(skel->arena->xor64_noreturn_value, 0x101ull << 32, "xor64_noreturn_value"); + ASSERT_EQ(skel->arena->xor32_noreturn_value, 0x101, "xor32_noreturn_value"); } static void test_cmpxchg(struct arena_atomics *skel) @@ -146,6 +164,27 @@ static void test_xchg(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xchg32_result, 1, "xchg32_result"); } +static void test_fetch_r0(struct arena_atomics *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + /* No need to attach it, just run it directly */ + prog_fd = bpf_program__fd(skel->progs.fetch_r0); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) + return; + if (!ASSERT_OK(topts.retval, "test_run_opts retval")) + return; + + ASSERT_EQ(skel->arena->fetch_src_r0_value, 0x11100000111ull, "fetch_src_r0_value"); + ASSERT_EQ(skel->arena->fetch_src_r0_result, 0x11000000110ull, "fetch_src_r0_result"); + + ASSERT_EQ(skel->arena->fetch_dst_r0_value, 0x11100000111ull, "fetch_dst_r0_value"); + ASSERT_EQ(skel->arena->fetch_dst_r0_result, 0x11000000110ull, "fetch_dst_r0_result"); + ASSERT_EQ(skel->arena->fetch_dst_r0_readback, 0x11100000111ull, "fetch_dst_r0_readback"); +} + static void test_uaf(struct arena_atomics *skel) { LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -160,6 +199,18 @@ static void test_uaf(struct arena_atomics *skel) return; ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails"); + +#ifndef __powerpc64__ + /* + * A fault clears the fetched register and leaves the address alone. The + * powerpc64 JIT does not clear the register of a faulting atomic yet. + */ + ASSERT_EQ(skel->arena->uaf_r1_after_dst_r0, 0, "uaf_r1_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_dst_r0, skel->arena->uaf_r0_before_dst_r0, + "uaf_r0_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_src_r0, 0, "uaf_r0_after_src_r0"); + ASSERT_EQ(skel->arena->uaf_reread, 0, "uaf_reread"); +#endif } static void test_load_acquire(struct arena_atomics *skel) @@ -256,6 +307,8 @@ void serial_test_arena_atomics(void) test_cmpxchg(skel); if (test__start_subtest("xchg")) test_xchg(skel); + if (test__start_subtest("fetch_r0")) + test_fetch_r0(skel); if (test__start_subtest("uaf")) test_uaf(skel); if (test__start_subtest("load_acquire")) diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 73bc2b835f3f..ce76a73d197b 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -91,13 +91,17 @@ int sub(const void *ctx) return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global and64_value = (0x110ull << 32); -_Atomic __u32 __arena_global and32_value = 0x110; -#else +/* + * A relaxed __c11_atomic_fetch_*() whose result is discarded is what selects + * the non-fetching insn, so each operation keeps a *_noreturn_value of each + * width to cover that lowering as well. + */ __u64 __arena_global and64_value = (0x110ull << 32); +__u64 __arena_global and64_result = 0; +_Atomic __u64 __arena_global and64_noreturn_value = (0x110ull << 32); __u32 __arena_global and32_value = 0x110; -#endif +__u32 __arena_global and32_result = 0; +_Atomic __u32 __arena_global and32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int and(const void *ctx) @@ -105,25 +109,22 @@ int and(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_and(&and64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_and(&and32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_and(&and64_value, 0x011ull << 32); - __sync_fetch_and_and(&and32_value, 0x011); -#endif + and64_result = __sync_fetch_and_and(&and64_value, 0x011ull << 32); + and32_result = __sync_fetch_and_and(&and32_value, 0x011); + __c11_atomic_fetch_and(&and64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_and(&and32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u32 __arena_global or32_value = 0x110; -_Atomic __u64 __arena_global or64_value = (0x110ull << 32); -#else +/* u32s first: .addr_space.1 must not have holes, see the clang-17 note below */ __u32 __arena_global or32_value = 0x110; +__u32 __arena_global or32_result = 0; +_Atomic __u32 __arena_global or32_noreturn_value = 0x110; __u64 __arena_global or64_value = (0x110ull << 32); -#endif +__u64 __arena_global or64_result = 0; +_Atomic __u64 __arena_global or64_noreturn_value = (0x110ull << 32); SEC("raw_tp/sys_enter") int or(const void *ctx) @@ -131,25 +132,21 @@ int or(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_or(&or64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_or(&or32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_or(&or64_value, 0x011ull << 32); - __sync_fetch_and_or(&or32_value, 0x011); -#endif + or64_result = __sync_fetch_and_or(&or64_value, 0x011ull << 32); + or32_result = __sync_fetch_and_or(&or32_value, 0x011); + __c11_atomic_fetch_or(&or64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_or(&or32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global xor64_value = (0x110ull << 32); -_Atomic __u32 __arena_global xor32_value = 0x110; -#else __u64 __arena_global xor64_value = (0x110ull << 32); +__u64 __arena_global xor64_result = 0; +_Atomic __u64 __arena_global xor64_noreturn_value = (0x110ull << 32); __u32 __arena_global xor32_value = 0x110; -#endif +__u32 __arena_global xor32_result = 0; +_Atomic __u32 __arena_global xor32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int xor(const void *ctx) @@ -157,13 +154,10 @@ int xor(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_xor(&xor64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_xor(&xor32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); - __sync_fetch_and_xor(&xor32_value, 0x011); -#endif + xor64_result = __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); + xor32_result = __sync_fetch_and_xor(&xor32_value, 0x011); + __c11_atomic_fetch_xor(&xor64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_xor(&xor32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; @@ -213,23 +207,96 @@ int xchg(const void *ctx) return 0; } +__u64 __arena_global fetch_src_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_src_r0_result = 0; +__u64 __arena_global fetch_dst_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_dst_r0_result = 0; +__u64 __arena_global fetch_dst_r0_readback = 0; + +/* + * A fetching OR with the operand in r0, and one with the arena pointer in r0. + * The x86 JIT needs RAX for its CMPXCHG loop and substitutes BPF_REG_AX for + * whichever of the two is r0, so both have to keep working. Hand-written + * because clang picks its own registers and will not reliably emit either. + */ +SEC("raw_tp/sys_enter") +int fetch_r0(const void *ctx) +{ + if (pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; +#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) + asm volatile ( + "r1 = %[fetch_src_r0_value] ll;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 0x1100000011 ll;" + ".8byte %[fetch_src_r0_insn];" + "r2 = %[fetch_src_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : + : __imm_addr(fetch_src_r0_value), + __imm_insn(fetch_src_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(fetch_src_r0_result) + : __clobber_all); + + asm volatile ( + "r0 = %[fetch_dst_r0_value] ll;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 0x1100000011 ll;" + ".8byte %[fetch_dst_r0_insn];" + "r2 = %[fetch_dst_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + /* r0 is only read by the atomic, so it must still be the pointer. */ + "r3 = *(u64 *)(r0 + 0);" + "r2 = %[fetch_dst_r0_readback] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : + : __imm_addr(fetch_dst_r0_value), + __imm_insn(fetch_dst_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(fetch_dst_r0_result), + __imm_addr(fetch_dst_r0_readback) + : __clobber_all); +#endif + + return 0; +} + __u64 __arena_global uaf_sink; volatile __u64 __arena_global uaf_recovery_fails; +/* Register state after the r0 forms below fault, checked by the test */ +__u64 __arena_global uaf_r1_after_dst_r0; +__u64 __arena_global uaf_r0_before_dst_r0; +__u64 __arena_global uaf_r0_after_dst_r0; +__u64 __arena_global uaf_r0_after_src_r0; +/* What reading the faulted address again gives, see the end of uaf() */ +__u64 __arena_global uaf_reread; SEC("syscall") int uaf(const void *ctx) { if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; +/* + * __BPF_FEATURE_ADDR_SPACE_CAST is required by the asm below, which spells + * addr_space_cast textually. That and the mnemonic both arrived in LLVM 19. + */ #if defined(ENABLE_ATOMICS_TESTS) && !defined(__TARGET_ARCH_arm64) && \ - !defined(__TARGET_ARCH_x86) + defined(__BPF_FEATURE_ADDR_SPACE_CAST) __u32 __arena *page32; __u64 __arena *page64; void __arena *page; page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); bpf_arena_free_pages(&arena, page, 1); - uaf_recovery_fails = 24; + uaf_recovery_fails = 26; + uaf_r1_after_dst_r0 = 1; + uaf_r0_after_dst_r0 = 1; + uaf_r0_after_src_r0 = 1; + uaf_reread = 1; page32 = (__u32 __arena *)page; uaf_sink += __sync_fetch_and_add(page32, 1); @@ -282,6 +349,74 @@ int uaf(const void *ctx) uaf_recovery_fails -= 1; uaf_sink += __sync_lock_test_and_set(page64, 1); uaf_recovery_fails -= 1; + + /* + * The x86 JIT needs RAX for the CMPXCHG loop it lowers a fetching + * AND/OR/XOR into, and substitutes BPF_REG_AX for r0 when r0 is either + * operand. Only then does the fault fixup have to name that register, + * so spell both forms out: clang picks its own and never produces them. + * + * The freed page is the first of the arena, whose 32-bit address is 0, + * which would make "the address register was left alone" an empty + * check. Use the page after it, which is unmapped as well. + */ + asm volatile ( + "r0 = %[page];" + "r0 += 4096;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 1;" + "r2 = %[uaf_r0_before_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + ".8byte %[fetch_dst_r0];" + "r2 = %[uaf_r0_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + "r2 = %[uaf_r1_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + : : [page]"r"(page64), + __imm_insn(fetch_dst_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(uaf_r0_before_dst_r0), + __imm_addr(uaf_r0_after_dst_r0), + __imm_addr(uaf_r1_after_dst_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 1;" + ".8byte %[fetch_src_r0];" + "r2 = %[uaf_r0_after_src_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : : [page]"r"(page64), + __imm_insn(fetch_src_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(uaf_r0_after_src_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + /* + * If a fault were recovered by the arena mapping its zeroed scratch + * page rather than by the JIT's exception table entry, the access above + * would have succeeded and written there. The page has to still be + * unmapped, so that the read faults and gives 0. + */ + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r3 = *(u64 *)(r1 + 0);" + "r2 = %[uaf_reread] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : : [page]"r"(page64), + __imm_addr(uaf_reread) + : "r1", "r2", "r3", "memory"); #endif return 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c index bebc37f501b7..ecfc6873ffa9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -7,6 +7,7 @@ void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, __u64 flags) __ksym; +void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; #ifdef __TARGET_ARCH_arm64 #define ARENA_VM_START (1ull << 32) @@ -35,6 +36,7 @@ struct { void __kfunc_btf_root(void) { bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_arena_free_pages(0, 0, 0); } /* Tests start with r6 = address of a new page as the user space sees it, a number */ @@ -256,6 +258,260 @@ __naked void atomics(void) : __clobber_all); } +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 99; \ + goto 9f; \ +1: r0 = 1; \ + if r1 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r1 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ + r0 = 2; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r1 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ + r0 = 3; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ + r0 = 5; \ + if r1 != 0x110 goto 9f; \ + r1 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ + r0 = 6; \ + if r1 != 0x111 goto 9f; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ + r0 = 7; \ + if r1 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor with the operand in r0") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r0 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 2; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r0 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 5; \ + if r2 != 0x110 goto 9f; \ + r0 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 6; \ + if r2 != 0x111 goto 9f; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 7; \ + if r2 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)) + : __clobber_all); +} + +/* + * The page is gone, so the access faults. The fetched register has to come back + * 0 and everything else has to be left alone, r0 included. Reading the address + * again has to give 0 as well: had the fault been recovered by the arena mapping + * its zeroed scratch page instead of by the JIT's exception table entry, the + * access would have succeeded and left a 1 there. + */ +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or through a number to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 1; \ + goto 9f; \ +1: r0 = 2; \ + if r1 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 3; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or with the operand in r0 to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_r0_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 2; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)) + : __clobber_all); +} + SEC("syscall") __arena_scalar __description("arena_scalar: cmpxchg through r0") From 9f9c30435ca92cddb37e67cda46ec02e26dd0a07 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:30:44 -0700 Subject: [PATCH 541/660] bpf, x86: Pass a trampoline's stack arguments by the calling convention A trampoline on a function of a program that can tail call keeps the tail call counter pointer by pushing rax after allocating its frame. When it calls the original function, for fexit, fmod_ret or fsession, the push sits between the stack arguments and the return address, which the x86-64 calling convention does not allow. For example, in run() -> call_stack() -> callee_stack(a, b, c, d, e, f, g, h) run() -> subprog_tail() -> bpf_tail_call() callee_stack() is not on the path to the tail call, so the verifier allows its stack arguments g and h. fexit on it builds, with T the trampoline's rbp: T-120 h save_args() T-128 g save_args(), RSP after sub rsp, 128 T-136 tail_call_cnt_ptr push rax, RSP at the call T-144 return address call orig_call and callee_stack() finds the counter pointer where g belongs, at 8(%rsp) on entry, and g where h belongs. Keep the pointer in a slot of the frame instead, so that the stack arguments sit right above the return address again: T-120 tail_call_cnt_ptr mov [rbp - 120], rax T-128 (pad to 16) T-136 h save_args() T-144 g save_args(), RSP at the call T-152 return address call orig_call Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments") Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183044.1132751-1-yonghong.song@linux.dev --- arch/x86/net/bpf_jit_comp.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 004e2ddeb4f6..64bd35409ec7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -4233,9 +4233,9 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ -#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack) \ - __LOAD_TCC_PTR(-round_up(stack, 8) - 8) +/* mov rax, qword ptr [rbp - off] */ +#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(off) \ + __LOAD_TCC_PTR(-(off)) /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); @@ -4305,6 +4305,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im { int i, ret, nr_regs = m->nr_args, stack_size = 0; int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; + int tcc_off = 0; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; @@ -4355,11 +4356,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * * RBP - run_ctx_off [ bpf_tramp_run_ctx ] * + * RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG * [ ... ] * [ stack_arg2 ] * RBP - arg_stack_off [ stack_arg1 ] - * RSP [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX */ /* room for return value of orig_call or fentry prog */ @@ -4390,6 +4392,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; run_ctx_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { /* the space that used to pass arguments on-stack */ stack_size += (nr_regs - get_nr_used_regs(m)) * 8; @@ -4442,7 +4449,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* sub rsp, stack_size */ emit_sub_rsp(&prog, stack_size); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - EMIT1(0x50); /* push rax */ + /* mov QWORD PTR [rbp - tcc_off], rax */ + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -tcc_off); /* mov QWORD PTR [rbp - rbx_off], rbx */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); @@ -4503,7 +4511,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Before calling the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } if (flags & BPF_TRAMP_F_ORIG_STACK) { @@ -4567,7 +4575,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Before running the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } /* restore return value of orig_call or fentry prog back into RAX */ From 6b26307ce697732224a5bfb26d77e25734b9d5d3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:30:49 -0700 Subject: [PATCH 542/660] selftests/bpf: Test trampoline stack args in a program that can tail call Test the previous patch with a target program made of two call chains: run() -> call_stack() -> callee_stack(a, b, c, d, e, f, g, h) run() -> subprog_tail() -> bpf_tail_call() callee_stack() takes eight u64 arguments, so g and h are passed on the stack. Because run() can reach a tail call, a trampoline on any function of the program keeps the tail call counter pointer. callee_stack() itself does not lead to the tail call, which is why the verifier still lets it take stack arguments. Attach fentry and fexit programs to callee_stack(). The fentry program checks that it sees g and h, and the test checks the value that callee_stack() returns when the fexit trampoline calls it. Without the previous patch, the function takes the counter pointer for its first stack argument: test_stack_arg_tailcall:FAIL:stack ret unexpected stack ret: actual -359789677662885869 != expected 204 test_stack_arg_tailcall:FAIL:stack fexit ret unexpected stack fexit ret: actual -359789677662885869 != expected 204 Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183049.1133231-1-yonghong.song@linux.dev --- .../selftests/bpf/prog_tests/tracing_struct.c | 59 +++++++++++++++++++ .../progs/tracing_stack_arg_tailcall_tgt.c | 59 +++++++++++++++++++ .../progs/tracing_stack_arg_tailcall_tracer.c | 23 ++++++++ 3 files changed, 141 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c create mode 100644 tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 15b95d0235b5..8086586d3e19 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -5,6 +5,8 @@ #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" #include "tracing_struct_int128.skel.h" +#include "tracing_stack_arg_tailcall_tgt.skel.h" +#include "tracing_stack_arg_tailcall_tracer.skel.h" static void test_struct_args(void) { @@ -146,6 +148,61 @@ destroy_skel: #endif } +/* + * A trampoline on a function of a tail_call_reachable program keeps the tail + * call counter pointer on its stack, and has to hand the function its stack + * arguments past it. + */ +static void test_stack_arg_tailcall(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_stack_arg_tailcall_tracer *tracer = NULL; + struct tracing_stack_arg_tailcall_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_stack_arg_tailcall_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_stack_arg_tailcall_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.stack_entry, fd, "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_stack_arg_tailcall_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_stack_arg_tailcall_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_h, 8, "stack:h"); + ASSERT_EQ(tgt->bss->ret_stack, 204, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 204, "stack fexit ret"); + +out: + tracing_stack_arg_tailcall_tracer__destroy(tracer); + tracing_stack_arg_tailcall_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -181,6 +238,8 @@ void test_tracing_struct(void) test_struct_many_args(); if (test__start_subtest("int128_args")) test_int128_args(); + if (test__start_subtest("stack_arg_tailcall")) + test_stack_arg_tailcall(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c new file mode 100644 index 000000000000..5fde710c4e37 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__u64 base; +__u64 ret_stack; + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* g and h are on the stack, for the x86-64 kernel convention too. */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, __u64 h) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + h * 8; +} + +/* No stack args of its own, so it can share a program with a tail call. */ +static __noinline __u64 call_stack(__u64 x) +{ + return callee_stack(x + 1, x + 2, x + 3, x + 4, x + 5, x + 6, x + 7, + x + 8); +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +/* Makes the program tail_call_reachable, though callee_stack() is not. */ +static __noinline int subprog_tail(void *ctx) +{ + int ret = 0; + + bpf_tail_call_static(ctx, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = call_stack(base); +#endif + return subprog_tail(ctx); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c new file mode 100644 index 000000000000..1800e7b7afe2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +long stack_g, stack_h, stack_ret; + +SEC("fentry/callee_stack") +int stack_entry(unsigned long long *ctx) +{ + stack_g = ctx[6]; + stack_h = ctx[7]; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[8]; + return 0; +} + +char _license[] SEC("license") = "GPL"; From 98e46955064b1f4013e3e163f9691055b40e4385 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:30:54 -0700 Subject: [PATCH 543/660] selftests/bpf: Test by-value __int128 kfunc arguments at an odd slot This follows up the series "bpf: Support by-value struct and __int128 arguments" [1]. The __int128 kfunc tests it added pass the __int128 at an even argument slot, after two u64s, or past the argument registers, where it lands on an even arm64 stack slot. So neither the AAPCS64 rule that rounds the register number up to an even one for an argument aligned to 16 bytes, nor the same rule for a stack slot, is exercised. Each leaves a hole the arm64 JIT has to open by moving the __int128 and every argument after it up a slot. Add three kfuncs. Like the earlier ones, each weighs its arguments by argument slot, so that two eightbytes landing in each other's slots are caught as well as one landing in a hole: - bpf_kfunc_call_test_i128_arg_odd(): the __int128 at an odd register slot. - bpf_kfunc_call_test_i128_arg_odd_many(): the same, with arguments after it that arm64 shifts onto its stack, and a second __int128 at an odd arm64 stack slot. - bpf_kfunc_call_test_i128_arg_stack_hole(): a struct holding an __int128 at an odd stack slot on both x86-64 and arm64, after x86-64 gives R9 back to an argument following a 16-byte struct. x86-64 has no register alignment rule and places the first two like the BPF convention, so they check that its JIT leaves them where they are. The last two need more than the five BPF argument registers and are only built with a compiler that supports stack arguments. On arm64 the first two kfuncs get BTF only with the pahole patch "dwarf_loader: Skip the argument register the arm64 ABI leaves as a hole" [2], in pahole master after v1.32. An older pahole finds the argument after the __int128 in an unexpected register, x4 rather than x3, and leaves the function out of BTF, so the tests fail to load there. Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260912195156.980886-1-yonghong.song@linux.dev/ [1] Link: https://lore.kernel.org/dwarves/20260917164043.1577508-1-yonghong.song@linux.dev/ [2] Link: https://patch.msgid.link/20261016183054.1134162-1-yonghong.song@linux.dev --- .../selftests/bpf/progs/aggregate_arg_kfunc.c | 71 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 26 +++++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 13 ++++ 3 files changed, 110 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c index 52d3d5d53946..c87e886423f4 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c @@ -50,6 +50,27 @@ int aggregate_arg_kfunc_int128(struct __sk_buff *skb) return 0; } +/* + * arm64 rounds the register number up to an even one for an argument + * aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1 + * empty, and the last argument in x4. The x86-64 ABI has no such rule. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2) + return 1; + + return 0; +} + #endif /* __SIZEOF_INT128__ */ #if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) @@ -131,6 +152,56 @@ int aggregate_arg_kfunc_split8(struct __sk_buff *skb) #ifdef __SIZEOF_INT128__ +/* + * The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that + * arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots, + * and onto its stack. The second __int128 then leaves the second arm64 + * stack slot empty. x86-64 packs the eightbytes like the BPF convention, + * so nothing moves there. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + u128 w = ((u128)~a << 64) | ~b; + + if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) != + 1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 + + 9 * ~b + 10 * ~a) + return 1; + + return 0; +} + +/* + * x86-64 moves s to the stack and gives R9 back to the seventh argument. + * w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot + * empty before it: the fourth on x86-64, the second on arm64. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b }; + + if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) != + 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 + + 10 * ~b + 11 * ~a) + return 1; + + return 0; +} + /* * Both conventions pad the stack to align this __int128, and the BPF * convention pads for neither, so both JITs move it up an eightbyte. diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index b4f0af2aeca4..00641d597f95 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1047,6 +1047,29 @@ __bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; } +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd(u64 a, __int128 v, u64 b) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd_many(u64 a, __int128 v, u64 b, u64 c, + u64 d, u64 e, u64 f, __int128 w) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + + b * 4 + c * 5 + d * 6 + e * 7 + f * 8 + + (u64)w * 9 + (u64)((unsigned __int128)w >> 64) * 10; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_stack_hole(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, + u64 f, u64 g, + struct prog_test_i128_arg w) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + + f * 8 + g * 9 + (u64)w.v * 10 + + (u64)((unsigned __int128)w.v >> 64) * 11; +} + __bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, __int128 v) { @@ -1770,6 +1793,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd_many) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_stack_hole) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 67c02a421d13..a6a93c901416 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -66,6 +66,12 @@ struct prog_test_pair_arg { /* 16 bytes: two argument registers */ __u64 hi; }; +#ifdef __SIZEOF_INT128__ +struct prog_test_i128_arg { /* 16 bytes, aligned to 16 like its member */ + __int128 v; +}; +#endif + struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ void *p; __u64 x; @@ -234,6 +240,13 @@ struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; __u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; #ifdef __SIZEOF_INT128__ __u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd(__u64 a, __int128 v, __u64 b) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd_many(__u64 a, __int128 v, __u64 b, __u64 c, + __u64 d, __u64 e, __u64 f, __int128 w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_stack_hole(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct prog_test_pair_arg s, + __u64 f, __u64 g, + struct prog_test_i128_arg w) __ksym; __u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __int128 v) __ksym; #endif From ab70d0d7f1b1a9e7948ca49940519c0475eb6de4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:00 -0700 Subject: [PATCH 544/660] bpf: Describe a BPF function to its trampoline as one u64 per slot A trampoline on a BPF function, fentry, fexit or fsession attached through attach_prog_fd, takes the function's arguments from where BPF code left them. A call from one BPF function to another passes slot N where the kernel convention passes its Nth u64 argument, in a register or a stack slot, whatever the types of the arguments: the x86-64, arm64 and riscv JITs all place a stack argument that way. The model of the function, distilled from its BTF, still carries those types, so each arch places the arguments by its own rules for them. For a 16-byte argument at an odd slot, such as s in u64 f(u64 a, struct { __int128 v; } s, u64 b) the arm64 trampoline reads s from x2 and x3, where the caller left it in x1 and x2, and b from x4. And an x86-64 trampoline puts a 16-byte argument at slot 6 wholly on the stack, while x86-64 BPF code splits it between r9 and the stack. The s390 trampoline, which takes a 16-byte argument by reference, loads s through its first half as if that were a pointer, and saves and restores only three of the four registers the caller passed. Rewrite the model of a target program's function in bpf_check_attach_target() as one u64 argument per argument slot. Every convention places N u64 arguments in slot order, so all of the above take them from where BPF code put them, and no arch has to know that the function is a BPF one. The model no longer gives the size and flags of each argument, so an arch that sign-extends a narrow signed argument, or clears the unused bytes of a narrow stack argument, for the program does not do that for a BPF function any more: the program gets the eightbyte BPF code passed, as it does on x86-64, and fexit hands it back unchanged. Fixes: afdbb4cdd3d2 ("bpf, arm64: Place trampoline arguments by the arm64 calling convention") Fixes: 4dd676ca5b7a ("bpf: Support by-value struct arguments up to 16 bytes") Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183100.1134528-1-yonghong.song@linux.dev --- kernel/bpf/verifier.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 62f8951126ba..434013b38f17 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22006,6 +22006,32 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type) } } +/* + * BPF code passes argument slot N where the kernel convention passes its Nth + * u64 argument, so describe a BPF function to its trampoline as one u64 per + * slot, and every arch places its arguments right. + */ +static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, + struct btf_func_model *m) +{ + u32 i, nslots = 0; + + for (i = 0; i < m->nr_args; i++) + nslots += btf_func_model_arg_slots(m, i); + if (nslots > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "The function %s has %u argument slots. Too many.\n", + tname, nslots); + return -EINVAL; + } + + for (i = 0; i < nslots; i++) { + m->arg_size[i] = sizeof(u64); + m->arg_flags[i] = 0; + } + m->nr_args = nslots; + return 0; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -22279,6 +22305,11 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret; + if (tgt_prog) { + ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); + if (ret < 0) + return ret; + } if (tgt_info->fmodel.ret_size > 8 && attach_uses_trampoline_retval(prog->expected_attach_type)) { From 0e6d7e9669be0cba0041689c44d42ce4e62c4139 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:05 -0700 Subject: [PATCH 545/660] bpf, x86: Place trampoline arguments by the x86-64 calling convention save_args() numbers the on-stack argument slots consecutively, and the area built for the original function is sized the same way, while the x86-64 ABI rounds the stack slot up to an even one for an argument aligned to 16 bytes: an __int128, or an aggregate holding one. When such an argument lands on an odd stack slot, it and every argument after it are read from the wrong place, so a bpf program attached to the function reads a neighbouring eightbyte, and the arguments handed on to the original function are shifted. For example: /* s holds an __int128 */ void foo(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, struct s h, u64 i) x86-64 puts g in the first stack slot, leaves the second empty and puts h in the third and fourth, while the trampoline reads h from the second and third. Take the position of each argument slot from bpf_jit_place_args(), which knows the x86-64 rules, backfill included. save_args() then reads each slot from its register or incoming stack slot, restore_regs() puts back only the register slots, and the area built for the original function mirrors the incoming one, holes included. A trampoline on a BPF function is not changed by this: since the previous patch, the function's model has one u64 per argument slot, which bpf_jit_place_args() places in slot order, so the trampoline still takes the arguments where BPF code put them. Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183105.1134967-1-yonghong.song@linux.dev --- arch/x86/net/bpf_jit_comp.c | 187 ++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 64bd35409ec7..11a4e4f87aac 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2359,6 +2359,16 @@ static const u8 x86_arg_reg[] = { BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, }; +static bool x86_arg_on_stack(u8 pos) +{ + return pos >= x86_arg_abi.nr_arg_regs; +} + +static s32 x86_arg_stack_off(u8 pos) +{ + return (pos - x86_arg_abi.nr_arg_regs) * 8; +} + /* * Move the arguments the x86-64 ABI places somewhere other than the argument * slot the BPF calling convention gave them. @stack_base addresses the @@ -3883,21 +3893,17 @@ static void clean_stack_garbage(const struct btf_func_model *m, } } -/* get the count of the regs that are used to pass arguments */ -static int get_nr_used_regs(const struct btf_func_model *m) +/* the eightbytes taken by the on-stack arguments, alignment holes included */ +static int get_nr_stack_slots(const u8 *pos_of_slot, int nr_slots) { - int i, arg_regs, nr_used_regs = 0; + int slot; - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_used_regs + arg_regs <= 6) - nr_used_regs += arg_regs; - - if (nr_used_regs >= 6) - break; + for (slot = nr_slots - 1; slot >= 0; slot--) { + if (x86_arg_on_stack(pos_of_slot[slot])) + return x86_arg_stack_off(pos_of_slot[slot]) / 8 + 1; } - return nr_used_regs; + return 0; } /* @@ -3927,20 +3933,21 @@ static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base } static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags, - u64 arena_base) + const u8 *pos_of_slot, int stack_size, + bool for_call_origin, u32 flags, u64 arena_base) { - int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; + int arg_regs, first_off = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24; - int i, j; + int bargs_off = stack_size; + int i, j, slot, off, doff; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; @@ -3962,8 +3969,36 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * * the arg1-5,arg7 will be passed by regs, and arg6 will * by stack. + * + * An argument aligned to 16 bytes, an __int128 or a struct + * holding one, also starts at an even stack slot, which can + * leave a hole before it. bpf_jit_place_args() worked out + * where each eightbyte is. */ - if (nr_regs + arg_regs > 6) { + for (j = 0; j < arg_regs; j++, slot++, bargs_off -= 8) { + u8 pos = pos_of_slot[slot]; + u32 src; + + if (!x86_arg_on_stack(pos)) { + /* + * Only copy the arguments on-stack to + * current 'stack_size' and ignore the regs, + * used to prepare the arguments on-stack for + * origin call. + */ + if (for_call_origin) + continue; + + /* copy the arguments from regs into stack */ + src = x86_arg_reg[pos]; + if (arena_arg) { + emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -bargs_off); + continue; + } + /* copy function arguments from origin stack frame * into current stack frame. * @@ -3973,53 +4008,33 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * entered through the fentry call, so rbp + 24, and * a single one when it is entered with a jmp or * called indirectly, so rbp + 16. + * + * The area for the origin call mirrors the incoming + * one, hole and all; only the bpf program takes the + * arguments packed. */ - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, - nr_stack_slots * 8 + stack_args_off); - if (arena_arg) - emit_arena_arg_conv(prog, BPF_REG_0, nullable, - (u32)arena_base); - emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, - -stack_size); + off = x86_arg_stack_off(pos); + doff = for_call_origin ? stack_size - off : bargs_off; + emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, + stack_args_off + off); + if (arena_arg) + emit_arena_arg_conv(prog, BPF_REG_0, nullable, + (u32)arena_base); + emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -doff); - if (!nr_stack_slots) - first_off = stack_size; - stack_size -= 8; - nr_stack_slots++; - } - } else { - /* Only copy the arguments on-stack to current - * 'stack_size' and ignore the regs, used to - * prepare the arguments on-stack for origin call. - */ - if (for_call_origin) { - nr_regs += arg_regs; - continue; - } - - /* copy the arguments from regs into stack */ - for (j = 0; j < arg_regs; j++) { - u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; - - if (arena_arg) { - emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); - src = BPF_REG_0; - } - emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); - stack_size -= 8; - nr_regs++; - } + if (!nr_stack_slots) + first_off = doff; + nr_stack_slots++; } } clean_stack_garbage(m, prog, nr_stack_slots, first_off); } -static void restore_regs(const struct btf_func_model *m, u8 **prog, +static void restore_regs(u8 **prog, const u8 *pos_of_slot, int nr_slots, int stack_size) { - int i, j, arg_regs, nr_regs = 0; + int slot; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: @@ -4028,23 +4043,10 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, * * The logic here is similar to what we do in save_args() */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_regs + arg_regs <= 6) { - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - BPF_REG_FP, - -stack_size); - stack_size -= 8; - nr_regs++; - } - } else { - stack_size -= 8 * arg_regs; - } - - if (nr_regs >= 6) - break; + for (slot = 0; slot < nr_slots; slot++, stack_size -= 8) { + if (!x86_arg_on_stack(pos_of_slot[slot])) + emit_ldx(prog, BPF_DW, x86_arg_reg[pos_of_slot[slot]], + BPF_REG_FP, -stack_size); } } @@ -4303,7 +4305,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_nodes *tnodes, void *func_addr) { - int i, ret, nr_regs = m->nr_args, stack_size = 0; + int i, ret, nr_regs, stack_size = 0; + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; int tcc_off = 0; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; @@ -4327,12 +4330,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im arena_base = bpf_tramp_arena_base(m, tnodes, flags); - for (i = 0; i < m->nr_args; i++) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - - /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 - * are passed through regs, the remains are through stack. + /* + * x86-64 supports up to MAX_BPF_FUNC_ARGS argument slots, six in + * registers and the rest on the stack, where bpf_jit_place_args() + * puts them. */ + nr_regs = bpf_jit_place_args(&x86_arg_abi, m, pos_of_slot); if (nr_regs > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; @@ -4344,11 +4347,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or * BPF_TRAMP_F_RET_FENTRY_RET flags * - * [ reg_argN ] always - * [ ... ] - * RBP - regs_off [ reg_arg1 ] program's ctx pointer + * [ arg slotN ] always, packed, whether + * [ ... ] in a register or on the + * RBP - regs_off [ arg slot1 ] stack; program's ctx pointer * - * RBP - func_meta_off [ regs count, etc ] always + * RBP - func_meta_off [ slot count, etc ] always * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * @@ -4358,10 +4361,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * * RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX * - * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG - * [ ... ] - * [ stack_arg2 ] - * RBP - arg_stack_off [ stack_arg1 ] + * [ stack slotN ] BPF_TRAMP_F_CALL_ORIG, + * [ ... ] the incoming stack + * [ stack slot2 ] arguments, alignment + * RBP - arg_stack_off [ stack slot1 ] holes included */ /* room for return value of orig_call or fentry prog */ @@ -4372,7 +4375,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += nr_regs * 8; regs_off = stack_size; - /* function matedata, such as regs count */ + /* function matedata, such as the argument slot count */ stack_size += 8; func_meta_off = stack_size; @@ -4399,7 +4402,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { /* the space that used to pass arguments on-stack */ - stack_size += (nr_regs - get_nr_used_regs(m)) * 8; + stack_size += get_nr_stack_slots(pos_of_slot, nr_regs) * 8; /* make sure the stack pointer is 16-byte aligned if we * need pass arguments on stack, which means * [stack_size + 8(rbp) + 8(rip) + 8(origin rip)] @@ -4455,7 +4458,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); func_meta = nr_regs; - /* Store number of argument registers of the traced function */ + /* Store number of argument slots of the traced function */ emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (flags & BPF_TRAMP_F_IP_ARG) { @@ -4463,7 +4466,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags, arena_base); + save_args(m, &prog, pos_of_slot, regs_off, false, flags, arena_base); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -4504,8 +4507,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags, 0); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); + save_args(m, &prog, pos_of_slot, arg_stack_off, true, flags, 0); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the @@ -4558,7 +4561,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, regs_off); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be From b68782116bfdcdd05949a662c14d5fa644d10ab8 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:10 -0700 Subject: [PATCH 546/660] selftests/bpf: Test trampoline args around a 16-byte aligned stack arg Add two bpf_testmod functions whose 16-byte aligned argument, a struct holding an __int128, is passed on the stack after a hole: - bpf_testmod_test_int128_stack(): seven u64s before the struct, so it leaves a register empty on arm64 and a stack slot empty on x86-64. - bpf_testmod_test_int128_backfill(): a 16-byte struct of two longs moves to the x86-64 stack and gives R9 back to the next argument; the struct then leaves a stack slot empty on both x86-64 and arm64. The __int128 is wrapped in a struct because btf_ctx_access() refuses a program reading an integer argument wider than eight bytes, while it lets the program read each slot of a struct argument. The fentry programs read each argument slot from ctx[] and check that the trampoline packs them in the BPF convention, and the fexit programs check the return value, weighted by argument slot, which is only right if the original function got its arguments back where its convention wants them. Without the preceding x86 fix the test fails with: test_int128_args:FAIL:s:h.lo unexpected s:h.lo: actual 2 != expected 8 ... test_int128_args:FAIL:s ret unexpected s ret: actual 285 != expected 385 Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183110.1135597-1-yonghong.song@linux.dev --- .../selftests/bpf/prog_tests/tracing_struct.c | 22 +++++++-- .../bpf/progs/tracing_struct_int128.c | 45 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 43 ++++++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 8086586d3e19..a2da219e1476 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -119,9 +119,11 @@ static void test_int128_args(void) { /* * __int128 arguments are passed in a register pair on x86_64 and - * arm64, which the trampoline packs into two context slots. Other - * architectures pass a __int128 differently (e.g. s390x passes larger - * arguments by reference), so only exercise this on x86_64 and arm64. + * arm64, which the trampoline packs into two context slots. Past the + * registers, both start an argument aligned to 16 bytes at an even + * stack slot, which the trampoline packs too. Other architectures + * pass a __int128 differently (e.g. s390x passes larger arguments by + * reference), so only exercise this on x86_64 and arm64. */ #if defined(__x86_64__) || defined(__aarch64__) struct tracing_struct_int128 *skel; @@ -141,6 +143,20 @@ static void test_int128_args(void) ASSERT_EQ(skel->bss->t_c, 3, "t:c"); ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + ASSERT_EQ(skel->bss->s_g, 7, "s:g"); + ASSERT_EQ(skel->bss->s_h_lo, 8, "s:h.lo"); + ASSERT_EQ(skel->bss->s_h_hi, 9, "s:h.hi"); + ASSERT_EQ(skel->bss->s_i, 10, "s:i"); + ASSERT_EQ(skel->bss->s_ret, 385, "s ret"); + + ASSERT_EQ(skel->bss->b_f_a, 6, "b:f.a"); + ASSERT_EQ(skel->bss->b_f_b, 7, "b:f.b"); + ASSERT_EQ(skel->bss->b_g, 8, "b:g"); + ASSERT_EQ(skel->bss->b_h, 9, "b:h"); + ASSERT_EQ(skel->bss->b_i_lo, 10, "b:i.lo"); + ASSERT_EQ(skel->bss->b_i_hi, 11, "b:i.hi"); + ASSERT_EQ(skel->bss->b_ret, 506, "b ret"); + destroy_skel: tracing_struct_int128__destroy(skel); #else diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c index 4638dfec1f38..5520ee500c01 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -15,4 +15,49 @@ int test_int128_arg_fexit(unsigned long long *ctx) return 0; } +/* + * The arguments come packed in the BPF convention whatever holes the + * kernel convention left between them: an eightbyte each, and two for the + * struct holding an __int128. + */ +long s_g, s_h_lo, s_h_hi, s_i, s_ret; + +SEC("fentry/bpf_testmod_test_int128_stack") +int test_int128_stack_fentry(unsigned long long *ctx) +{ + s_g = ctx[6]; + s_h_lo = ctx[7]; + s_h_hi = ctx[8]; + s_i = ctx[9]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_stack") +int test_int128_stack_fexit(unsigned long long *ctx) +{ + s_ret = ctx[10]; + return 0; +} + +long b_f_a, b_f_b, b_g, b_h, b_i_lo, b_i_hi, b_ret; + +SEC("fentry/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fentry(unsigned long long *ctx) +{ + b_f_a = ctx[5]; + b_f_b = ctx[6]; + b_g = ctx[7]; + b_h = ctx[8]; + b_i_lo = ctx[9]; + b_i_hi = ctx[10]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fexit(unsigned long long *ctx) +{ + b_ret = ctx[11]; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 00641d597f95..38edb4addc5b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -197,6 +197,43 @@ bpf_testmod_test_int128_arg(__int128 a, int b, long c) bpf_testmod_test_struct_arg_result = (long)a + b + c; return bpf_testmod_test_struct_arg_result; } + +struct bpf_testmod_struct_int128 { + __int128 a; +}; + +/* + * x86-64 puts g in the first stack slot and h, aligned to 16 bytes, in + * the third and fourth, leaving a hole. arm64 leaves x7 empty and puts h + * in the first two stack slots. Either way h and i are not where the BPF + * convention packs them, right after g. + */ +noinline long +bpf_testmod_test_int128_stack(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, struct bpf_testmod_struct_int128 h, u64 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f * 6 + g * 7 + (u64)h.a * 8 + + (u64)((unsigned __int128)h.a >> 64) * 9 + i * 10; + return bpf_testmod_test_struct_arg_result; +} + +/* + * x86-64 has no register left for f after e, so f goes to the stack and g + * takes R9 back; h goes to the third stack slot and i, aligned to 16 + * bytes, to the fifth and sixth. arm64 has registers up to g, puts h in + * the first stack slot and i in the third and fourth. + */ +noinline long +bpf_testmod_test_int128_backfill(u64 a, u64 b, u64 c, u64 d, u64 e, + struct bpf_testmod_struct_arg_2 f, u64 g, u64 h, + struct bpf_testmod_struct_int128 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f.a * 6 + f.b * 7 + g * 8 + h * 9 + (u64)i.a * 10 + + (u64)((unsigned __int128)i.a >> 64) * 11; + return bpf_testmod_test_struct_arg_result; +} #endif __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) @@ -683,6 +720,12 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, #ifdef __SIZEOF_INT128__ (void)bpf_testmod_test_int128_ret(i); (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); + (void)bpf_testmod_test_int128_stack(1, 2, 3, 4, 5, 6, 7, + (struct bpf_testmod_struct_int128){ .a = ((__int128)9 << 64) | 8 }, + 10); + (void)bpf_testmod_test_int128_backfill(1, 2, 3, 4, 5, + (struct bpf_testmod_struct_arg_2){ .a = 6, .b = 7 }, 8, 9, + (struct bpf_testmod_struct_int128){ .a = ((__int128)11 << 64) | 10 }); #endif (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); From 9c6fc12a8e4e249758d9ca6559123b24078af34e Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:15 -0700 Subject: [PATCH 547/660] selftests/bpf: Test trampoline args of a BPF function at an odd slot Attach fentry and fexit programs to two functions of another BPF program, each taking a struct holding an __int128 at an odd argument slot: - callee_odd(): the struct after a single u64, where arm64 would leave a register empty. - callee_stack(): the struct after seven u64s, where x86-64 would leave a stack slot empty and arm64 a register. BPF code calls them with the arguments packed in the BPF convention, so that is where the trampoline has to take them from. The fentry programs take the arguments with BPF_PROG2(), which reads each from its slots in ctx[], and the fexit programs check the return value, weighted by slot, which is only right if the function got its arguments back where its caller put them. Both take a struct by value, which GCC passes by reference instead, so they are only built with clang, and callee_stack() only with a clang that supports stack arguments. Unlike a kernel function taking an __int128, which s390x passes by reference, nothing in the test depends on how the arch passes a 16-byte value, so it runs on s390x as well as on x86_64 and arm64. BPF_PROG2() reads the struct from its two context slots the way BPF code passed it, so its halves come out right on big-endian s390x too. The s390 JIT does not take stack arguments, so callee_stack() is only built for x86 and arm64, and only callee_odd() is traced on s390x. Describing the functions to the trampoline by the types of their arguments, rather than as one u64 per slot, the x86-64 trampoline fails the test with: test_int128_tgt_prog:FAIL:stack:h.lo unexpected stack:h.lo: actual 9 != expected 8 test_int128_tgt_prog:FAIL:stack:h.hi unexpected stack:h.hi: actual 10 != expected 9 Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183115.1135762-1-yonghong.song@linux.dev --- .../selftests/bpf/prog_tests/tracing_struct.c | 74 +++++++++++++++++++ .../bpf/progs/tracing_struct_int128_tgt.c | 71 ++++++++++++++++++ .../bpf/progs/tracing_struct_int128_tracer.c | 50 +++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c create mode 100644 tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index a2da219e1476..a02731bc6573 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -5,6 +5,8 @@ #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" #include "tracing_struct_int128.skel.h" +#include "tracing_struct_int128_tgt.skel.h" +#include "tracing_struct_int128_tracer.skel.h" #include "tracing_stack_arg_tailcall_tgt.skel.h" #include "tracing_stack_arg_tailcall_tracer.skel.h" @@ -164,6 +166,76 @@ destroy_skel: #endif } +static void test_int128_tgt_prog(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_struct_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.odd_entry, fd, "callee_odd"); + err = err ?: bpf_program__set_attach_target(tracer->progs.odd_exit, fd, "callee_odd"); + if (tgt->rodata->has_stack_arg) { + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_entry, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, + "callee_stack"); + } else { + bpf_program__set_autoload(tracer->progs.stack_entry, false); + bpf_program__set_autoload(tracer->progs.stack_exit, false); + } + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_struct_int128_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->odd_a, 1, "odd:a"); + ASSERT_EQ(tracer->bss->odd_lo, 8, "odd:v.lo"); + ASSERT_EQ(tracer->bss->odd_hi, 9, "odd:v.hi"); + ASSERT_EQ(tracer->bss->odd_b, 2, "odd:b"); + ASSERT_EQ(tgt->bss->ret_odd, 52, "odd ret"); + ASSERT_EQ(tracer->bss->odd_ret, 52, "odd fexit ret"); + + if (tgt->rodata->has_stack_arg) { + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_lo, 8, "stack:h.lo"); + ASSERT_EQ(tracer->bss->stack_hi, 9, "stack:h.hi"); + ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i"); + ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret"); + } + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + /* * A trampoline on a function of a tail_call_reachable program keeps the tail * call counter pointer on its stack, and has to hand the function its stack @@ -254,6 +326,8 @@ void test_tracing_struct(void) test_struct_many_args(); if (test__start_subtest("int128_args")) test_int128_args(); + if (test__start_subtest("int128_tgt_prog")) + test_int128_tgt_prog(); if (test__start_subtest("stack_arg_tailcall")) test_stack_arg_tailcall(); if (test__start_subtest("union_args")) diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c new file mode 100644 index 000000000000..60e997470b59 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +struct int128_arg { + __int128 v; +}; + +__u64 base; +__u64 ret_odd, ret_stack; + +#if defined(__clang__) + +const volatile bool has_struct_arg = true; + +/* arm64 would want v in x2 and x3, and b in x4. */ +__noinline __u64 callee_odd(__u64 a, struct int128_arg v, __u64 b) +{ + return a + (__u64)v.v * 2 + (__u64)((unsigned __int128)v.v >> 64) * 3 + + b * 4; +} + +#else + +const volatile bool has_struct_arg = false; + +#endif + +#if defined(__clang__) && \ + (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* + * x86-64 would want h in its third and fourth stack slots, and arm64 in + * its first and second, leaving x7 empty. + */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, + struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (__u64)h.v * 8 + (__u64)((unsigned __int128)h.v >> 64) * 9 + + i * 10; +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) + struct int128_arg v = { .v = ((__int128)(base + 9) << 64) | (base + 8) }; + + ret_odd = callee_odd(base + 1, v, base + 2); +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4, + base + 5, base + 6, base + 7, v, base + 10); +#endif +#endif + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c new file mode 100644 index 000000000000..5843dc5af9cf --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include + +struct int128_arg { + __int128 v; +}; + +long odd_a, odd_lo, odd_hi, odd_b, odd_ret; + +SEC("fentry/callee_odd") +int BPF_PROG2(odd_entry, __u64, a, struct int128_arg, v, __u64, b) +{ + odd_a = a; + odd_lo = (__u64)v.v; + odd_hi = (unsigned __int128)v.v >> 64; + odd_b = b; + return 0; +} + +SEC("fexit/callee_odd") +int odd_exit(unsigned long long *ctx) +{ + odd_ret = ctx[4]; + return 0; +} + +long stack_g, stack_lo, stack_hi, stack_i, stack_ret; + +SEC("fentry/callee_stack") +int BPF_PROG2(stack_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + __u64, f, __u64, g, struct int128_arg, h, __u64, i) +{ + stack_g = g; + stack_lo = (__u64)h.v; + stack_hi = (unsigned __int128)h.v >> 64; + stack_i = i; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[10]; + return 0; +} + +char _license[] SEC("license") = "GPL"; From 9b2e5136760a675202622459d242ac803f11d8d5 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:20 -0700 Subject: [PATCH 548/660] bpf: Refuse a trampoline on an unreliable static function with stack args When the BTF of a function in the target program is unreliable, the model of the function given to its trampoline is MAX_BPF_FUNC_REG_ARGS u64 arguments. The verifier takes a static function's stack arguments from its BTF before it finds the BTF unreliable, though, so such a function can still take more: one with six or more arguments, one of them a plain pointer, for example. The trampoline saves and restores five registers then. After an fentry program, the rest of the register arguments, r9 on x86-64 and x5 to x7 on arm64, hold whatever __bpf_prog_exit() left there, and for fexit the stack arguments are not copied either, so the function goes on with garbage in arguments the verifier checked, pointers included. Refuse the attach when such a function has more argument slots in its BTF than fit in registers. The main program is left alone: it takes only its context, and the verifier gives it no stack arguments when its BTF is unreliable. Fixes: 0f6bd5e7a804 ("bpf: Support stack arguments for bpf functions") Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183120.1136008-1-yonghong.song@linux.dev --- kernel/bpf/verifier.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 434013b38f17..ddfa0c6d2755 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22032,6 +22032,33 @@ static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tnam return 0; } +/* + * The verifier lets a static function use stack arguments counted from its + * BTF, even if that BTF turns out unreliable. A trampoline on the function is + * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the + * stack arguments, so such a function cannot be attached to. + */ +static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, + const struct btf_type *func_proto, const char *tname) +{ + struct btf_func_model m; + u32 i, nslots = 0; + + if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { + /* at least a slot for each argument */ + nslots = btf_type_vlen(func_proto); + } else { + for (i = 0; i < m.nr_args; i++) + nslots += btf_func_model_arg_slots(&m, i); + } + if (nslots <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", + tname); + return -EINVAL; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -22299,8 +22326,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL; - if (tgt_prog && conservative) + if (tgt_prog && conservative) { + if (subprog > 0) { + ret = check_unreliable_func_args(log, btf, t, tname); + if (ret) + return ret; + } t = NULL; + } ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) From 8395b40318816ded8e4c55f07741dd5594ee9b9b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 16 Oct 2026 11:31:25 -0700 Subject: [PATCH 549/660] selftests/bpf: Test trampolines on more BPF functions with stack args Add two functions to the BPF target program of the int128_tgt_prog test. callee_split() takes a struct holding an __int128 after five u64s, so on x86-64 it is passed split between r9 and the stack. fentry and fexit programs check that the trampoline takes it from there and hands it back there. callee_unreliable() takes six u64-sized arguments, the first a plain pointer, which makes its BTF unreliable while the sixth is still passed on the stack. The new tgt_prog_unreliable subtest checks that an fentry program cannot be attached to it. Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016183125.1136468-1-yonghong.song@linux.dev --- .../selftests/bpf/prog_tests/tracing_struct.c | 62 +++++++++++++++++++ .../bpf/progs/tracing_struct_int128_tgt.c | 32 +++++++++- .../bpf/progs/tracing_struct_int128_tracer.c | 27 ++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index a02731bc6573..de358f335d98 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -194,9 +194,15 @@ static void test_int128_tgt_prog(void) "callee_stack"); err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_entry, fd, + "callee_split"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_exit, fd, + "callee_split"); } else { bpf_program__set_autoload(tracer->progs.stack_entry, false); bpf_program__set_autoload(tracer->progs.stack_exit, false); + bpf_program__set_autoload(tracer->progs.split_entry, false); + bpf_program__set_autoload(tracer->progs.split_exit, false); } if (!ASSERT_OK(err, "set_attach_target")) goto out; @@ -226,6 +232,13 @@ static void test_int128_tgt_prog(void) ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i"); ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret"); ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret"); + + ASSERT_EQ(tracer->bss->split_e, 5, "split:e"); + ASSERT_EQ(tracer->bss->split_lo, 8, "split:h.lo"); + ASSERT_EQ(tracer->bss->split_hi, 9, "split:h.hi"); + ASSERT_EQ(tracer->bss->split_i, 10, "split:i"); + ASSERT_EQ(tgt->bss->ret_split, 246, "split ret"); + ASSERT_EQ(tracer->bss->split_ret, 246, "split fexit ret"); } out: @@ -236,6 +249,53 @@ out: #endif } +/* + * A static function whose BTF is unreliable gets a trampoline that only knows + * of the register arguments, so one taking stack arguments must be refused. + */ +static void test_tgt_prog_unreliable(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + char log_buf[4096] = {}; + struct bpf_program *prog; + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + bpf_object__for_each_program(prog, tracer->obj) + bpf_program__set_autoload(prog, false); + bpf_program__set_autoload(tracer->progs.unreliable_entry, true); + bpf_program__set_log_buf(tracer->progs.unreliable_entry, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(tracer->progs.unreliable_entry, fd, + "callee_unreliable"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + ASSERT_ERR(err, "tracer__load"); + ASSERT_HAS_SUBSTR(log_buf, "unreliable BTF and stack arguments", "log"); + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + /* * A trampoline on a function of a tail_call_reachable program keeps the tail * call counter pointer on its stack, and has to hand the function its stack @@ -328,6 +388,8 @@ void test_tracing_struct(void) test_int128_args(); if (test__start_subtest("int128_tgt_prog")) test_int128_tgt_prog(); + if (test__start_subtest("tgt_prog_unreliable")) + test_tgt_prog_unreliable(); if (test__start_subtest("stack_arg_tailcall")) test_stack_arg_tailcall(); if (test__start_subtest("union_args")) diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c index 60e997470b59..2561422c25dd 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c @@ -8,7 +8,7 @@ struct int128_arg { }; __u64 base; -__u64 ret_odd, ret_stack; +__u64 ret_odd, ret_stack, ret_split, ret_unreliable; #if defined(__clang__) @@ -46,6 +46,28 @@ static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, i * 10; } +/* + * x86-64 BPF code passes h split between r9 and its first stack slot; + * arm64 passes it in x5 and x6, so nothing is split there. + */ +static __noinline __u64 callee_split(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + (__u64)h.v * 6 + + (__u64)((unsigned __int128)h.v >> 64) * 7 + i * 8; +} + +/* + * A plain pointer makes the BTF of a static function unreliable, but f is + * still passed on the stack. The store keeps p a pointer. + */ +static __noinline __u64 callee_unreliable(__u64 *p, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f) +{ + *p += 1; + return *p + b * 2 + c * 3 + d * 4 + e * 5 + f * 6; +} + #else const volatile bool has_stack_arg = false; @@ -63,6 +85,14 @@ int run(void *ctx) defined(__BPF_FEATURE_STACK_ARGUMENT) ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4, base + 5, base + 6, base + 7, v, base + 10); + ret_split = callee_split(base + 1, base + 2, base + 3, base + 4, + base + 5, v, base + 10); + { + __u64 x = base; + + ret_unreliable = callee_unreliable(&x, base + 2, base + 3, + base + 4, base + 5, base + 6); + } #endif #endif return 0; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c index 5843dc5af9cf..fc81c2eea4a8 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c @@ -47,4 +47,31 @@ int stack_exit(unsigned long long *ctx) return 0; } +long split_e, split_lo, split_hi, split_i, split_ret; + +SEC("fentry/callee_split") +int BPF_PROG2(split_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + struct int128_arg, h, __u64, i) +{ + split_e = e; + split_lo = (__u64)h.v; + split_hi = (unsigned __int128)h.v >> 64; + split_i = i; + return 0; +} + +SEC("fexit/callee_split") +int split_exit(unsigned long long *ctx) +{ + split_ret = ctx[8]; + return 0; +} + +/* Refused: the trampoline would lose the stack argument. */ +SEC("?fentry/callee_unreliable") +int unreliable_entry(unsigned long long *ctx) +{ + return 0; +} + char _license[] SEC("license") = "GPL"; From 458575e1f2ddea944d679fe778c92e370a9b61a4 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 16 Oct 2026 21:16:35 +0200 Subject: [PATCH 550/660] bpf: Don't echo user-supplied map info fields back to userspace Commit ea2e6467ac36 ("bpf: Return hashes of maps in BPF_OBJ_GET_INFO_BY_FD") started copying struct bpf_map_info in from userspace in order to read the new hash and hash_size input fields. Before that, the kernel ignored the contents of the user buffer and returned a zeroed structure with only the output fields filled in. Some output fields are only set for certain maps: btf_id, btf_key_type_id and btf_value_type_id when the map has BTF (and btf_vmlinux_id for struct_ops maps, ifindex, netns_dev and netns_ino when the map is offloaded). For all other maps, these fields now return whatever userspace passed in before. This breaks userspace which reuses one bpf_map_info for several queries. Restore the original behavior by zeroing the structure and copying in only hash and hash_size from userspace before the output fields are filled in. Fixes: ea2e6467ac36 ("bpf: Return hashes of maps in BPF_OBJ_GET_INFO_BY_FD") Reported-by: Quentin Monnet Reported-by: Tianyi Chen Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261016191635.662690-1-daniel@iogearbox.net --- kernel/bpf/syscall.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 654f896af265..0f27604a54aa 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5448,7 +5448,11 @@ static int bpf_map_get_info_by_fd(struct file *file, info_len = min_t(u32, sizeof(info), info_len); memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) + if (info_len >= offsetofend(struct bpf_map_info, hash) && + get_user(info.hash, &uinfo->hash)) + return -EFAULT; + if (info_len >= offsetofend(struct bpf_map_info, hash_size) && + get_user(info.hash_size, &uinfo->hash_size)) return -EFAULT; info.type = map->map_type; From b32b4956d451fa70d57d2639e3fdca71a7f9c65c Mon Sep 17 00:00:00 2001 From: Maxim Skokov Date: Fri, 16 Oct 2026 22:47:56 +0300 Subject: [PATCH 551/660] bpftool: Print ENOTSUPP as "Operation not supported" Some BPF commands return ENOTSUPP (524) to user space: iterating local storage maps, element operations on ringbuf maps, creating an LRU hash map without preallocation, freezing a struct_ops map, or running a program type that has no test_run. ENOTSUPP is kernel internal, so strerror() only gives "Unknown error 524": # bpftool map getnext pinned /sys/fs/bpf/sk_storage Error: can't get next key: Unknown error 524 Add a small helper that knows ENOTSUPP and use it where bpftool prints the error of map create, map element operations, map freeze and prog run. Suggested-by: Daniel Borkmann Signed-off-by: Maxim Skokov Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20261016194825.230433-2-skokovmaksimevg@gmail.com --- tools/bpf/bpftool/common.c | 11 +++++++++++ tools/bpf/bpftool/main.h | 1 + tools/bpf/bpftool/map.c | 18 +++++++++--------- tools/bpf/bpftool/prog.c | 2 +- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index f9b87efa7c9a..ca5fa2f0bf9b 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -72,6 +72,17 @@ void p_info(const char *fmt, ...) va_end(ap); } +/* + * Like strerror(), but also knows ENOTSUPP (524), which the kernel returns + * to user space from some BPF commands although it is not a userspace errno. + */ +const char *bpf_strerror(int err) +{ + if (err == 524 /* ENOTSUPP */) + return "Operation not supported"; + return strerror(err); +} + static bool is_bpffs(const char *path) { struct statfs st_fs; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 6851346a6030..b1eb4b5d991b 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -98,6 +98,7 @@ extern const char *cert_path; void __printf(1, 2) p_err(const char *fmt, ...); void __printf(1, 2) p_info(const char *fmt, ...); +const char *bpf_strerror(int err); bool is_prefix(const char *pfx, const char *str); int detect_common_prefix(const char *arg, ...); diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 20d59eab09a1..d809b075e734 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -228,7 +228,7 @@ print_entry_error(struct bpf_map_info *map_info, void *key, int lookup_errno) print_hex_data_json(key, map_info->key_size); jsonw_name(json_wtr, "value"); jsonw_start_object(json_wtr); /* error */ - jsonw_string_field(json_wtr, "error", strerror(lookup_errno)); + jsonw_string_field(json_wtr, "error", bpf_strerror(lookup_errno)); jsonw_end_object(json_wtr); /* error */ jsonw_end_object(json_wtr); /* entry */ } else { @@ -241,7 +241,7 @@ print_entry_error(struct bpf_map_info *map_info, void *key, int lookup_errno) msg = ""; print_entry_error_msg(map_info, key, - msg ? : strerror(lookup_errno)); + msg ? : bpf_strerror(lookup_errno)); } } @@ -1032,7 +1032,7 @@ static int do_update(int argc, char **argv) err = bpf_map_update_elem(fd, key, value, flags); if (err) { - p_err("update failed: %s", strerror(errno)); + p_err("update failed: %s", bpf_strerror(errno)); goto exit_free; } @@ -1136,7 +1136,7 @@ static int do_lookup(int argc, char **argv) printf("\n\nNot found\n"); } } else { - p_err("lookup failed: %s", strerror(errno)); + p_err("lookup failed: %s", bpf_strerror(errno)); } goto exit_free; @@ -1188,7 +1188,7 @@ static int do_getnext(int argc, char **argv) err = bpf_map_get_next_key(fd, key, nextkey); if (err) { - p_err("can't get next key: %s", strerror(errno)); + p_err("can't get next key: %s", bpf_strerror(errno)); goto exit_free; } @@ -1253,7 +1253,7 @@ static int do_delete(int argc, char **argv) err = bpf_map_delete_elem(fd, key); if (err) - p_err("delete failed: %s", strerror(errno)); + p_err("delete failed: %s", bpf_strerror(errno)); exit_free: free(key); @@ -1381,7 +1381,7 @@ offload_dev: fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, &attr); if (fd < 0) { - p_err("map create failed: %s", strerror(errno)); + p_err("map create failed: %s", bpf_strerror(errno)); goto exit; } @@ -1427,7 +1427,7 @@ static int do_pop_dequeue(int argc, char **argv) else printf("Error: empty map\n"); } else { - p_err("pop failed: %s", strerror(errno)); + p_err("pop failed: %s", bpf_strerror(errno)); } goto exit_free; @@ -1462,7 +1462,7 @@ static int do_freeze(int argc, char **argv) err = bpf_map_freeze(fd); close(fd); if (err) { - p_err("failed to freeze map: %s", strerror(errno)); + p_err("failed to freeze map: %s", bpf_strerror(errno)); return err; } diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index eb077236d2ea..43599c79c93a 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1499,7 +1499,7 @@ static int do_run(int argc, char **argv) err = bpf_prog_test_run_opts(fd, &test_attr); if (err) { - p_err("failed to run program: %s", strerror(errno)); + p_err("failed to run program: %s", bpf_strerror(errno)); goto free_ctx_out; } From 70ac4de43ad1d24ae0c845f35eed307442a0aabc Mon Sep 17 00:00:00 2001 From: Maxim Skokov Date: Fri, 16 Oct 2026 22:47:57 +0300 Subject: [PATCH 552/660] bpftool: Fail map dump for maps that can't be iterated "bpftool map dump" walks the map with bpf_map_get_next_key() and treats ENOENT as the end of the map. Map types that can't be iterated fail that call on the very first key: local storage and ringbuf with ENOTSUPP, bloom filter and arena with EOPNOTSUPP, queue and stack with EINVAL. bpftool then ends the walk without a word and prints what it prints for an empty map, "[]" with BTF or "Found 0 elements" without, and only the exit status tells that the map was not read. On a socket storage map that holds storage for a socket: # bpftool map dump pinned /sys/fs/bpf/sk_storage [] Ask for the first key before printing anything, and when that fails with anything but ENOENT, fail with the map type and the error instead of a dump: # bpftool map dump pinned /sys/fs/bpf/sk_storage Error: can't dump sk_storage map: Operation not supported # bpftool -j map dump pinned /sys/fs/bpf/sk_storage {"error":"can't dump sk_storage map: Operation not supported"} An error later in the walk is now reported too, as "bpftool map getnext" reports it, and the element count is printed only after a complete walk. Signed-off-by: Maxim Skokov Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20261016194825.230433-3-skokovmaksimevg@gmail.com --- tools/bpf/bpftool/map.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index d809b075e734..4b65edf74859 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -856,6 +856,21 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, } } + /* Fail early to avoid an empty dump if map type cannot be iterated */ + if (bpf_map_get_next_key(fd, NULL, key) && errno != ENOENT) { + const char *map_type_str, *err_str; + + err_str = bpf_strerror(errno); + map_type_str = libbpf_bpf_map_type_str(info->type); + if (map_type_str) + p_err("can't dump %s map: %s", map_type_str, err_str); + else + p_err("can't dump map of type %u: %s", info->type, + err_str); + err = -1; + goto exit_free; + } + if (wtr) { err = get_map_kv_btf(info, &btf); if (err) { @@ -883,8 +898,11 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, while (true) { err = bpf_map_get_next_key(fd, prev_key, key); if (err) { - if (errno == ENOENT) + if (errno == ENOENT) { err = 0; + break; + } + p_err("can't get next key: %s", bpf_strerror(errno)); break; } if (!dump_map_elem(fd, key, value, info, btf, wtr, @@ -897,7 +915,7 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, jsonw_end_array(wtr); /* elements */ if (show_header) jsonw_end_object(wtr); /* map object */ - } else { + } else if (!err) { printf("Found %u element%s\n", num_elems, num_elems != 1 ? "s" : ""); } From 631fee3f5a18b13f6d89df4e1c1146e71e420ea1 Mon Sep 17 00:00:00 2001 From: Maxim Skokov Date: Fri, 16 Oct 2026 22:47:58 +0300 Subject: [PATCH 553/660] selftests/bpf: Test bpftool map dump on maps that can't be iterated Socket storage can't be iterated from user space. Create a BPF_MAP_TYPE_SK_STORAGE map, attach storage to a socket, pin it, and check that "bpftool map dump" fails with nothing but an error: a JSON error object with -j, no elements on stdout in plain mode, and the reason on stderr. Socket storage always has BTF, so its plain dump goes through the BTF writer and never prints an element count. Check that path with a queue map created without BTF that holds one element: its plain dump must fail without a "Found" line, which bpftool printed as "Found 0 elements" before, and give the reason on stderr. Signed-off-by: Maxim Skokov Signed-off-by: Alexei Starovoitov Acked-by: Quentin Monnet Link: https://patch.msgid.link/20261016194825.230433-4-skokovmaksimevg@gmail.com --- .../bpf/prog_tests/bpftool_map_dump.c | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c new file mode 100644 index 000000000000..187b3f784307 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include "bpftool_helpers.h" + +#define PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_sk_storage" +#define QUEUE_PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_queue" +#define JSON_ERROR "{\"error\":\"can't dump sk_storage map" + +/* An empty socket storage map with an int value. */ +static int create_sk_storage_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NO_PREALLOC); + struct btf *btf; + int int_id, fd = -1; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto out; + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_key_type_id = int_id; + opts.btf_value_type_id = int_id; + fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage", sizeof(int), + sizeof(int), 0, &opts); + ASSERT_OK_FD(fd, "bpf_map_create"); +out: + btf__free(btf); + return fd; +} + +static void test_sk_storage(void) +{ + int map_fd, sk_fd = -1, value = 42, ret; + char output[1024] = {}; + + map_fd = create_sk_storage_map(); + if (map_fd < 0) + return; + + sk_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sk_fd, "socket")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST), + "add socket storage")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin map")) + goto out; + + /* + * Socket storage can't be iterated, so the dump must fail with an + * error and nothing else, rather than print an empty map that holds + * an element. The buffer is cleared before each run so that a run + * that fails to start can't match stale or uninitialised output. + */ + ret = get_bpftool_command_output("-j map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "json dump fails"); + ASSERT_STRNEQ(output, JSON_ERROR, strlen(JSON_ERROR), + "json dump is only the error"); + + memset(output, 0, sizeof(output)); + ret = get_bpftool_command_output("map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strchr(output, '['), "plain dump prints no elements"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, + "can't dump sk_storage map: Operation not supported", + "plain dump explains why on stderr"); + +out: + unlink(PIN_PATH); + if (sk_fd >= 0) + close(sk_fd); + close(map_fd); +} + +/* + * A queue map has no BTF, so its plain dump prints an element count + * rather than JSON. A dump that ignored the error would print "Found 0 + * elements" for a queue that holds one. + */ +static void test_queue(void) +{ + char output[1024] = {}; + int map_fd, value = 42, ret; + + map_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, "queue", 0, sizeof(int), 1, + NULL); + if (!ASSERT_OK_FD(map_fd, "bpf_map_create")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, NULL, &value, BPF_ANY), + "push to queue")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, QUEUE_PIN_PATH), "pin map")) + goto out; + + ret = get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strstr(output, "Found"), "plain dump prints no count"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, "can't dump queue map", + "plain dump explains why on stderr"); + +out: + unlink(QUEUE_PIN_PATH); + close(map_fd); +} + +void test_bpftool_map_dump(void) +{ + if (test__start_subtest("sk_storage")) + test_sk_storage(); + if (test__start_subtest("queue")) + test_queue(); +} From af873495cbcb066ab029f7f63b94410d4db47628 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Sat, 17 Oct 2026 22:38:12 +0800 Subject: [PATCH 554/660] bpf: Fix verifier WARN on narrow read of bpf_tcp_sock u64 fields A BPF program can read the middle of a u64 field in struct bpf_tcp_sock: tp = bpf_tcp_sock(listener); /* +0x54 is in the middle of the u64 bytes_received */ unsigned hi = *(unsigned *)((char *)tp + 0x54); bpf_tcp_sock_is_valid_access() allows this 4-byte read (the default case only checks size == 4), but bpf_tcp_sock_convert_ctx_access() only knows the field starts and has no idea how to handle such an offset. It emits no instruction, cnt stays 0 and the fixup pass trips verifier_bug(): verifier bug: error during ctx access conversion (0) Just disallow the narrower access by matching the whole u64 range of the two fields, so it is turned down at check time: invalid tcp_sock access off=84 size=4 Fixes: 655a51e536c0 ("bpf: Add struct bpf_tcp_sock and BPF_FUNC_tcp_sock") Reported-by: Kaiyan Mei Reported-by: Yinhao Hu Reported-by: Dongliang Mu Signed-off-by: Jiayuan Chen Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017143819.209510-1-jiayuan.chen@linux.dev Closes: https://lore.kernel.org/bpf/7a8ff335.1024c.1a14395f3e5.Coremail.kaiyanm@hust.edu.cn/ --- net/core/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index ba536be2915f..f0eae1c14b2f 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7600,8 +7600,8 @@ bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, return false; switch (off) { - case offsetof(struct bpf_tcp_sock, bytes_received): - case offsetof(struct bpf_tcp_sock, bytes_acked): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_received): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_acked): return size == sizeof(__u64); default: return size == sizeof(__u32); From 5256161e3573e0e35ac5e12f8ca904cf51e77f5b Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Sat, 17 Oct 2026 22:38:13 +0800 Subject: [PATCH 555/660] selftests/bpf: Test narrow read of bpf_tcp_sock u64 fields Add a test that reads the two u64 fields bytes_received and bytes_acked at offset + 4 with a u32 load, to make sure such a narrower access is rejected with "invalid tcp_sock access". Signed-off-by: Jiayuan Chen Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017143819.209510-2-jiayuan.chen@linux.dev --- .../selftests/bpf/progs/verifier_sock.c | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index 565c4d8bc3fa..d8e65938c35c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -611,6 +611,54 @@ l1_%=: r0 = *(u64*)(r0 + %[bpf_tcp_sock_bytes_acked__end]);\ : __clobber_all); } +SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_received") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_received_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_received) + 4) + : __clobber_all); +} + +SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_acked") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_acked_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_acked) + 4) + : __clobber_all); +} + SEC("cgroup/skb") __description("bpf_tcp_sock(bpf_sk_fullsock(skb->sk)): tp->snd_cwnd") __success __success_unpriv __retval(0) From b3ca872947ef322c475ceaa7a76a9ebb3356152e Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:03 +0000 Subject: [PATCH 556/660] bpf: Use the frame mem type in devmap egress XDP A devmap egress program that removes a whole fragment of a redirected xdp_frame with bpf_xdp_adjust_tail() or bpf_xdp_pull_data() frees it with page_frag_free() whatever the frame's memory type is. For a MEM_TYPE_PAGE_POOL frame from a page_pool based driver this drops the page reference owned by the pool while the pool still accounts the fragment ("page_pool leak" bad page, leaked pp reference). kernel/bpf/devmap.c:dev_map_bpf_prog_run() points xdp.rxq at an on-stack struct xdp_rxq_info rxq = { .dev = rx_dev }; whose mem.type is 0, i.e. MEM_TYPE_PAGE_SHARED, and net/core/filter.c:bpf_xdp_shrink_data() releases frags according to rxq->mem.type. cpumap, which runs programs on xdp_frames the same way, already copies xdpf->mem_type into its private rxq for each frame. With 9000 byte frames from virtio_net and an egress program that cuts the fragments off: BUG: Bad page state in process swapper/3 pfn:10660b page dumped because: page_pool leak bad_page+0xbd/0x190 __free_frozen_pages+0x5c7/0x900 bpf_xdp_adjust_tail+0x3be/0x480 bpf_prog_dad43f894a3faff7_egress+0x15c/0x1cd dev_map_bpf_prog_run+0x288/0x610 The pool waits for these pages for ever when its device is gone. That is the harmless case. A page of the pool can hold more than one fragment while its refcount is 1: the first page_frag_free() frees it under the fragment which is still in use, and the second one runs into page dumped because: VM_BUG_ON_PAGE(page_ref_count(page) == 0) kernel BUG at include/linux/mm.h:1846! veth is not affected. It takes references of its own on the pages and redirects the frame as MEM_TYPE_PAGE_SHARED. Fix by setting rxq.mem.type from each frame before running the devmap program, as cpu_map_bpf_prog_run_xdp() does. Fixes: ca9984c5f0ab ("bpf: devmap: provide rxq after redirect") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-2-alexei.starovoitov@gmail.com --- kernel/bpf/devmap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index dc7b859e8bbf..1b8f0d7f8455 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -350,6 +350,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, xdp_convert_frame_to_buff(xdpf, &xdp); xdp.txq = &txq; + rxq.mem.type = xdpf->mem_type; xdp.rxq = &rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp); From 9bdd5e7f5d8313fa1f3a210621b4007bb5ad0ec6 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:04 +0000 Subject: [PATCH 557/660] bpf: Forbid sleeping in hash map for_each callbacks A sleepable program (e.g. SEC("syscall"), lsm.s, fentry.s) can pass a callback to bpf_for_each_map_elem() on a hash map and call a sleepable helper or kfunc such as bpf_copy_from_user() from it. The verifier accepts this, but at run time the callback is invoked from inside an RCU read-side critical section: bpf_for_each_hash_elem() rcu_read_lock(); callback_fn(map, key, val, ctx, 0); bpf_copy_from_user() <- may fault and sleep rcu_read_unlock(); bpf_each_rhash_elem() does the same for BPF_MAP_TYPE_RHASH. Sleeping there triggers the "Voluntary context switch within RCU read-side critical section!" warning, delays grace periods, and with CONFIG_PREEMPT_RCU=n silently ends the read-side critical section that the bucket walk relies on. Synchronous callbacks are verified with the caller's state, so kernel/bpf/verifier.c:in_sleepable_context() only looks at the program's sleepability and the locks the program took itself, and does not know that the helper holds rcu_read_lock() around the callback. With a SEC("syscall") program whose callback reads a page that has to be read from disk first: BUG: sleeping function called from invalid context at include/linux/uaccess.h:169 RCU nest depth: 1, expected: 0 __might_fault+0x29/0x80 _copy_from_user+0x26/0x90 bpf_copy_from_user+0x23/0x60 bpf_prog_17d44fa39eebd6cc_cb+0x35/0x46 bpf_for_each_hash_elem+0xdb/0x280 Fix by marking the callback frame of hash map iteration as running under RCU and making in_sleepable_context() false while such a frame is on the call stack, so sleepable helpers, kfuncs and global functions are rejected there as inside bpf_rcu_read_lock(). The rules for trusting RCU protected pointers are left alone, which is conservative and keeps callbacks that verify today and do not sleep working. The flag is compared in func_states_equal() so that pruning never mixes the two contexts. Array map callbacks run without any lock and keep the caller's context. Fixes: 314ee05e2fc6 ("bpf: Add hashtab support for bpf_for_each_map_elem() helper") Fixes: 818e00848227 ("bpf: Implement iteration ops for resizable hashtab") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-3-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 5 +++++ kernel/bpf/states.c | 3 +++ kernel/bpf/verifier.c | 28 +++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 112ca1511da4..816948df8fff 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -379,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 953700822fa7..b3d250caaaa6 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -995,6 +995,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!old->no_stack_arg_load && cur->no_stack_arg_load) return false; + if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ddfa0c6d2755..883cf148235c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11530,6 +11530,19 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, if (err) return err; + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_RHASH: + /* Hash tables run the callback under rcu_read_lock(). */ + callee->in_rcu_callback_fn = true; + break; + default: + break; + } + callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; @@ -12273,10 +12286,22 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, return *ptr && (*ptr)->func ? 0 : -EINVAL; } +/* Is any frame on the current path a callback that runs under RCU? */ +static bool in_rcu_callback(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int i; + + for (i = 0; i <= state->curframe; i++) + if (state->frame[i]->in_rcu_callback_fn) + return true; + return false; +} + /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !in_rcu_cs(env); + return !in_rcu_cs(env) && !in_rcu_callback(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) @@ -12290,6 +12315,7 @@ static const char *non_sleepable_context_description(struct bpf_verifier_env *en return "IRQ-disabled region"; if (env->cur_state->active_locks) return "lock region"; + return "callback running under rcu_read_lock"; } return "non-sleepable prog"; } From f52b699c88634d0161c97247e0f791a840eb868d Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:05 +0000 Subject: [PATCH 558/660] bpf: Don't leak the inode when token creation fails bpf_token_create() allocates a bpffs inode and hands it to alloc_file_pseudo(). The inode reference only moves to the new file when that call succeeds: alloc_file_pseudo() leaves the caller's reference alone on failure, and FD_PREPARE() does not even evaluate the file expression when it cannot get a descriptor. Before the conversion to FD_PREPARE() the error path did iput(inode). The conversion dropped it, so each BPF_TOKEN_CREATE that fails with -EMFILE, -ENFILE or -ENOMEM at this point leaks an inode. A task that is allowed to create tokens can hit this at will by lowering RLIMIT_NOFILE, and the leaked inodes make umount of that bpffs instance complain about busy inodes. Put the inode again if no file was created. Fixes: 981bec8f698b ("bpf: convert bpf_token_create() to FD_PREPARE()") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-4-alexei.starovoitov@gmail.com --- kernel/bpf/token.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index e85a179523f0..19660ef95ddb 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -169,8 +169,11 @@ int bpf_token_create(union bpf_attr *attr) FD_PREPARE(fdf, O_CLOEXEC, alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops)); - if (fdf.err) + if (fdf.err) { + /* no file took over the inode reference */ + iput(inode); return fdf.err; + } token = kzalloc_obj(*token, GFP_USER); if (!token) From ccb88db8dac8fabde58656d5d4a33711e89c480a Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:06 +0000 Subject: [PATCH 559/660] bpf: Don't hand out a linked element as extra element When htab_map_update_elem() replaces an element of a preallocated map, alloc_htab_elem() takes the spare element of this cpu and puts the old element into the per-cpu slot right away, long before the old element is unlinked from its bucket. The bucket lock disables interrupts, but it does not keep out an NMI, or a tracing program on the return of alloc_htab_elem(). bpf_prog_active only helps if both programs are kprobe, tracepoint or perf_event programs. A program that updates an existing key in another bucket of the same map gets to use the slot: htab_map_update_elem(k1) tc, xdp, fentry, raw_tp, ... alloc_htab_elem() l_new = *pl_new *pl_new = l_old still linked in bucket 1 htab_map_update_elem(k2) perf_event program l_new = *pl_new the l_old of the outer update hlist_nulls_add_head_rcu(l_new, bucket 2) hlist_nulls_del_rcu(k2's old element) hlist_nulls_add_head_rcu(l_new, bucket 1) hlist_nulls_del_rcu(l_old) If l_old was the first element of bucket 1, the outer add points its pprev at l_new, and the delete takes it out of bucket 1 only. l_new->next now leads into bucket 2, where l_old stays with a poisoned pprev, on which a delete of k2 crashes. If it was not the first, the delete takes it out of bucket 2: k2 is gone from the map, and the chain of bucket 1 still contains l_old, whose next pointer leads into bucket 2. Either way a walk of bucket 1 ends on the nulls value of bucket 2. lookup_nulls_elem_raw() restarts whenever that happens, so every lookup in bucket 1 which does not find its key spins forever: rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: __htab_map_lookup_elem+0x354/0x4d0 BUG: KASAN: wild-memory-access in htab_map_delete_elem+0x30c/0x670 Write of size 8 at addr dead000000000122 by task extra/102 An NMI between the load and the store of the slot makes both updates use the same element, with much the same result. Empty the slot when the spare element is taken, in a way that is atomic on this cpu, and put the old element in only after it has been unlinked and its timers and the like have been cancelled. A nested update that finds the slot empty, which is the case that corrupted the map, fails with -EBUSY. Fixes: 8c290e60fa2a ("bpf: fix hashmap extra_elems logic") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-5-alexei.starovoitov@gmail.com --- kernel/bpf/hashtab.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9d4791ee1741..66ebb5a32807 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1279,11 +1279,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (prealloc) { if (old_elem) { /* if we're updating the existing element, - * use per-cpu extra elems to avoid freelist_pop/push + * use per-cpu extra elems to avoid freelist_pop/push. + * The caller puts old_elem into the slot once it is + * unlinked. Until then the slot is empty for an NMI + * or a kprobe that updates this map on this cpu. */ pl_new = this_cpu_ptr(htab->extra_elems); - l_new = *pl_new; - *pl_new = old_elem; + l_new = READ_ONCE(*pl_new); + do { + if (!l_new) + return ERR_PTR(-EBUSY); + } while (!try_cmpxchg_local(pl_new, &l_new, NULL)); } else { struct pcpu_freelist_node *l; @@ -1459,11 +1465,15 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, cancel - * its reusable special fields before it is available for reuse. + /* + * l_old replaces the extra element that alloc_htab_elem() took. + * Cancel its reusable special fields first: once it is in the + * slot, the next update on this cpu can reuse it. */ - if (htab_is_prealloc(htab)) + if (htab_is_prealloc(htab)) { check_and_cancel_fields(htab, l_old); + WRITE_ONCE(*this_cpu_ptr(htab->extra_elems), l_old); + } } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) From dfcd1765911223df60845e288ad392d5237f47a3 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:07 +0000 Subject: [PATCH 560/660] bpf: Reject sleepable programs for task_vma iterator A sleepable "iter.s/task_vma" program can be attached, but its ->show() callback runs with the mmap_lock of the iterated mm held for read. As soon as the program runs, bpf_iter_run_prog() calls might_fault(), and any bpf_copy_from_user() or bpf_copy_from_user_task() it does may really fault or take mmap_lock again. With lockdep this is reported as recursive locking of mmap_lock. Without lockdep, a fault on the reader's own mm, or access_process_vm() on the iterated mm, has to take mmap_lock for read again. If a writer queued up in the meantime that blocks behind the writer, which in turn waits for the read lock the iterator already holds, and the read() hangs: reader (iter.s/task_vma) other thread of the same mm task_vma_seq_get_next() mmap_read_lock(mm) mmap_write_lock(mm) <- waits bpf prog: copy_from_user() page fault mmap_read_lock(mm) <- queued behind the writer The reader waits killable, so the hang lasts until it gets a fatal signal. bpf_iter_link_attach() allows sleepable programs for the targets that set BPF_ITER_RESCHED, and task_vma sets it because cond_resched() between VMAs is fine with mmap_lock held. Rescheduling between objects and running a sleepable program inside ->show() are different properties, though, and task_vma only has the first one. The open-coded task_vma iterator does not have this problem since it hands out a snapshot of the VMA and holds no lock when it returns to the program. The vma handed to the program is only stable under mmap_lock, so the lock can't be dropped around ->show(). Add a feature flag BPF_ITER_NOSLEEP for targets which can't run sleepable programs although they can reschedule, and set it for task_vma. The check at attach time is not enough. BPF_LINK_UPDATE replaces the program of a link, and bpf_iter_link_replace() only compares the type, the attach type and the BTF id of the two programs, which are the same for a sleepable and a non-sleepable one. Do the check there as well. That also closes this way around the existing rule: so far a sleepable program could be put on a link of a target that doesn't set BPF_ITER_RESCHED, like the hash map element iterator, which runs it under rcu_read_lock(). BPF_LINK_CREATE of a sleepable program for the task_vma target, and BPF_LINK_UPDATE to one, now fail with -EINVAL, also for programs that don't touch user memory and have been working since sleepable iterator programs exist. The programs can still be loaded. They have to be loaded as non-sleepable programs, or use the open-coded iterator. Fixes: b77fb25dcb34 ("bpf: Add support for bpf iterator programs to use sleepable helpers") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-6-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 3 ++- kernel/bpf/bpf_iter.c | 17 ++++++++++++++--- kernel/bpf/task_iter.c | 6 +++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54144372281c..46d325aa178e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3137,7 +3137,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index b40eb404adab..fc3a75684ef7 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -72,6 +72,15 @@ static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_ return tinfo->reg_info->feature & BPF_ITER_RESCHED; } +/* Only allow sleepable program for resched-able iterator that can run one */ +static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, + const struct bpf_prog *prog) +{ + return !prog->sleepable || + (bpf_iter_target_support_resched(tinfo) && + !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); +} + static bool bpf_iter_support_resched(struct seq_file *seq) { struct bpf_iter_priv_data *iter_priv; @@ -409,6 +418,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog) { + struct bpf_iter_link *iter_link = + container_of(link, struct bpf_iter_link, link); int ret = 0; mutex_lock(&link_mutex); @@ -419,7 +430,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, if (link->prog->type != new_prog->type || link->prog->expected_attach_type != new_prog->expected_attach_type || - link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { + link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || + !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { ret = -EINVAL; goto out_unlock; } @@ -544,8 +556,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, if (!tinfo) return -ENOENT; - /* Only allow sleepable program for resched-able iterator */ - if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (!bpf_iter_target_support_prog(tinfo, prog)) return -EINVAL; link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 13e1aabe6f88..cf2bf388e181 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -737,7 +737,11 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", .attach_target = bpf_iter_attach_task, - .feature = BPF_ITER_RESCHED, + /* + * ->show() runs with the mmap_lock of the iterated mm held, which a + * sleepable program takes again when it faults on user memory. + */ + .feature = BPF_ITER_RESCHED | BPF_ITER_NOSLEEP, .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__task_vma, task), From c6dc0a288d4e787c578d7ab0464b9c943ad88ecc Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:08 +0000 Subject: [PATCH 561/660] bpf: Forbid sleeping in bpf_find_vma() callbacks A sleepable program can call bpf_find_vma() and use a sleepable helper such as bpf_copy_from_user() or bpf_copy_from_user_task() in the callback. The verifier accepts this, but bpf_find_vma() runs the callback with the mmap_lock of the mm held for read. If the user copy faults on that mm, the fault handler has to take mmap_lock again. Lockdep reports recursive locking, and if a writer queued up in between, the fault blocks behind the writer, which waits for the read lock the helper already holds: sleepable prog other thread of the same mm bpf_find_vma(current, ...) mmap_read_trylock(mm) mmap_write_lock(mm) <- waits callback: copy_from_user() page fault mmap_read_lock(mm) <- queued behind the writer This is the same problem as with sleepable task_vma iterator programs. The verifier checks synchronous callbacks in the caller's context, so in_sleepable_context() is true in the callback of a sleepable program. Verify the callback like a hash map for_each callback, as running in an RCU read-side critical section where sleeping is not allowed, and actually hold rcu_read_lock() around it, so that global functions the callback calls, which are then verified for a non-sleepable context, see the RCU protection they are verified with. Non-sleepable programs already run the callback under RCU, nothing changes for them. A sleepable program whose bpf_find_vma() callback calls any helper, kfunc or global function that may sleep is now rejected by the verifier, including ones that would be harmless under mmap_lock as they don't touch user memory. Fixes: 7c7e3d31e785 ("bpf: Introduce helper bpf_find_vma") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-7-alexei.starovoitov@gmail.com --- kernel/bpf/task_iter.c | 6 ++++++ kernel/bpf/verifier.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index cf2bf388e181..2fa6b8dae8c4 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -810,8 +810,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, vma = find_vma(mm, start); if (vma && vma->vm_start <= start && vma->vm_end > start) { + /* + * The callback must not sleep while mmap_lock is held, and the + * verifier checks it as an RCU read-side critical section. + */ + rcu_read_lock(); callback_fn((u64)(long)task, (u64)(long)vma, (u64)(long)callback_ctx, 0, 0); + rcu_read_unlock(); ret = 0; } bpf_mmap_unlock_mm(work, mm); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 883cf148235c..430d5309eb5c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11629,6 +11629,8 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ + callee->in_rcu_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; } From ee4f44e0e3c8125eebe9ac967b6984bc9253d489 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:09 +0000 Subject: [PATCH 562/660] bpf: Look at all 64 bits of a BPF_JSET mask on 32-bit In the branch of a BPF_JSET which is taken, the verifier can learn something about the register only if the mask has a single bit: then this bit is set. regs_refine_cond_op() tests this with is_power_of_2(val) and then ORs val into var_off. val is a u64, but is_power_of_2() takes an unsigned long. On a 32-bit kernel it only sees the lower half of the mask. For if r1 & 0xffffffff80000000 goto l which is a BPF_JSET | BPF_K with an immediate of 0x80000000, the lower half is a power of two. So the verifier takes all 33 bits for set at l, while the branch is taken if any of them is. A mask of 0x100000001 from a register does the same for bit 0. The log of an i386 kernel, for r6 = bpf_get_prandom_u32() << 31: 3: (45) if r6 & 0x80000000 goto pc+2 from 3 to 6: R6=0xffffffff80000000 R10=fp0 6: (77) r6 >>= 63 ; R6=1 When the program runs, r6 >> 63 is 0. >From then on the verifier works with known bits which the register does not have at run time. It accepts, for example, an offset into a map value which it derives from these bits and believes to be constant, and the program can access memory outside of the map value. Count the bits of the whole value. Only 32-bit kernels are affected. On a 64-bit kernel unsigned long holds the whole mask, and this patch changes nothing. Fixes: 960ea056561a ("bpf: verifier: teach the verifier to reason about the BPF_JSET instruction") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-8-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 430d5309eb5c..d0ba7156ba4f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18145,7 +18145,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor * bit 1 is set, which we can readily use in adjustments. */ - if (!is_power_of_2(val)) + if (hweight64(val) != 1) break; if (is_jmp32) { t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); From 12d07c38e1b0f79e48a37abd7103194212f2da8c Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:10 +0000 Subject: [PATCH 563/660] bpf: Don't let bpf_skb_adjust_room() use up the MSS When bpf_skb_adjust_room() grows a TCP GSO packet without BPF_F_ADJ_ROOM_FIXED_GSO, gso_size is lowered by the number of bytes added. Nothing looks at how large it is. gso_size is the size of the segments the peer has sent if the packet comes out of GRO, and what the sender has written into the virtio header if it comes from a tun device or a packet socket. If it equals len_diff the result is 0, and a packet of up to 64 KB is no GSO packet any more. If it is smaller, the 16 bit value wraps around. One less than len_diff gives GSO_BY_FRAGS. skb_segment() then takes the length of each segment from the frag_list. A packet with more than 64 KB of payload gets that far, and one from a tun device has no frag_list: BUG: KASAN: null-ptr-deref in skb_segment+0xcf1/0x1a90 BUG: kernel NULL pointer dereference, address: 0000000000000070 tcp_gso_segment+0x1be/0x9f0 inet_gso_segment+0x288/0x740 inet_gso_segment+0x288/0x740 skb_mac_gso_segment+0x179/0x2e0 What is left must not be less than TCP_MIN_GSO_SIZE either. The stack relies on that for the number of segments to fit into gso_segs, see commit c27c449d455a ("virtio-net: Ensure that TCP packets don't overflow gso_segs"). Refuse to grow a packet that would end up with less, with the error that other packets which cannot be adjusted get. Fixes: 2be7e212d541 ("bpf: add bpf_skb_adjust_room helper") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-9-alexei.starovoitov@gmail.com --- net/core/filter.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/net/core/filter.c b/net/core/filter.c index f0eae1c14b2f..8167cbadc625 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3612,6 +3612,14 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, return -ENOTSUPP; } + /* + * The MSS is lowered by len_diff. Leave the least that the stack lets a + * TCP GSO packet have, or gso_segs can overflow. + */ + if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) && + skb_shinfo(skb)->gso_size < len_diff + TCP_MIN_GSO_SIZE) + return -ENOTSUPP; + ret = skb_cow_head(skb, meta_len + len_diff); if (unlikely(ret < 0)) return ret; From 3ccd700543f16a2b2ab656f35dcaeb6bd7c45e22 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:11 +0000 Subject: [PATCH 564/660] bpf: Read the probe of a perf event under event_mutex For a perf event on a kprobe or uprobe event which was made in tracefs, bpf_get_kprobe_info() and bpf_get_uprobe_info() look the probe up by name in the list of dynamic events, and hand a pointer to its symbol or file name back to the caller. None of the callers holds event_mutex. The perf event keeps its trace event alive, nothing else. Other dynamic events are removed from the list and freed during the walk. And a probe which shares its event with other probes can be deleted while the event is in use, so even the probe which is found can be freed, together with the string, before the caller has measured and copied it. The result is a use-after-free in the walk, in strlen() and in the copy to user space, or a NULL pointer dereference. The readers are BPF_OBJ_GET_INFO_BY_FD and fdinfo of a perf link, for whoever holds a descriptor of the link, and BPF_TASK_FD_QUERY. The other side is anybody who may write to dynamic_events, kprobe_events or uprobe_events. Take event_mutex in bpf_get_perf_event_info() and give the caller a copy of the name, which it has to free. Fixes: 41bdc4b40ed6 ("bpf: introduce bpf subcommand BPF_TASK_FD_QUERY") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-10-alexei.starovoitov@gmail.com --- kernel/bpf/syscall.c | 17 +++++++++++------ kernel/trace/bpf_trace.c | 13 +++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0f27604a54aa..0f5eb55d49d6 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4020,19 +4020,19 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, *ulenp = 1; } if (!uname) - return 0; + goto out; if (buf) { err = bpf_copy_to_user(uname, buf, ulen, len); - if (err) - return err; } else { char zero = '\0'; if (put_user(zero, uname)) - return -EFAULT; + err = -EFAULT; } - return 0; +out: + kfree_const(buf); + return err; } #ifdef CONFIG_KPROBE_EVENTS @@ -4089,6 +4089,7 @@ static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event, name, offset, missed, addr, type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4142,6 +4143,7 @@ static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event, name, offset, ref_ctr_offset, type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4240,6 +4242,7 @@ static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event, "event_type:\t%s\n" "cookie:\t%llu\n", name, "tracepoint", event->bpf_cookie); + kfree_const(name); } static void bpf_probe_link_show_fdinfo(const struct perf_event *event, @@ -5767,11 +5770,13 @@ static int bpf_task_fd_query(const union bpf_attr *attr, err = bpf_get_perf_event_info(event, &prog_id, &fd_type, &buf, &probe_offset, &probe_addr, NULL); - if (!err) + if (!err) { err = bpf_task_fd_query_copy(attr, uattr, prog_id, fd_type, buf, probe_offset, probe_addr); + kfree_const(buf); + } goto put_file; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 5465e7eba238..304b426bd44d 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2209,6 +2209,7 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link * return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); } +/* On success the caller releases *buf with kfree_const() */ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, u32 *fd_type, const char **buf, u64 *probe_offset, u64 *probe_addr, @@ -2218,6 +2219,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, struct bpf_prog *prog; int flags, err = 0; + /* + * The list of dynamic events is walked, and a probe which has siblings + * can be deleted, with its strings, while its event is in use. + */ + guard(mutex)(&event_mutex); + prog = event->prog; if (!prog) return -ENOENT; @@ -2258,6 +2265,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, #endif } + if (!err && *buf) { + *buf = kstrdup_const(*buf, GFP_KERNEL); + if (!*buf) + err = -ENOMEM; + } + return err; } From 30a140c9d479974021f5c3eecb64415c96419c3f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:12 +0000 Subject: [PATCH 565/660] bpf: Don't saturate owner_refcnt on the last unpin bpf_local_storage_map_free() pins the owner with refcount_inc_not_zero() while it uncharges an element. The pin counts when bpf_local_storage_destroy() could not get local_storage->lock: destroy() then goes on, drops the owner's reference and spins until the count is zero. So the pin is the last reference whenever it is needed, and refcount_dec() takes that for a bug: it warns "decrement hit 0; leaking memory" and saturates the count. Unless destroy() happens to read the count between the two stores it spins for ever, under rcu_read_lock(). Drop the pin with refcount_dec_and_test(). Fixes: 5d800f87d0a5 ("bpf: Support lockless unlink when freeing map or local storage") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-11-alexei.starovoitov@gmail.com --- include/linux/bpf_local_storage.h | 13 +++++++++++++ kernel/bpf/bpf_local_storage.c | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 9e4f5c45c974..6c643ac51079 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -104,6 +104,19 @@ struct bpf_local_storage { refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ }; +/* + * Drop a reference got by refcount_inc_not_zero(&local_storage->owner_refcnt). + * bpf_local_storage_destroy() may have dropped the owner's reference meanwhile + * and wait for the count to reach zero, so this one can be the last. + */ +static inline void +bpf_local_storage_unpin_owner(struct bpf_local_storage *local_storage) +{ + bool last __maybe_unused; + + last = refcount_dec_and_test(&local_storage->owner_refcnt); +} + /* U16_MAX is much more than enough for sk local storage * considering a tcp_sock is ~2k. */ diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 4642da062f0f..7721834e4bff 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -199,7 +199,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * } if (pin_owner) - refcount_dec(&local_storage->owner_refcnt); + bpf_local_storage_unpin_owner(local_storage); } /* local_storage->lock must be held and selem->local_storage == local_storage. @@ -753,8 +753,8 @@ u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage) while (refcount_read(&local_storage->owner_refcnt)) cpu_relax(); /* - * Paired with refcount_dec() in bpf_selem_unlink_nofail() - * to make sure destroy() sees the correct local_storage->mem_charge. + * Paired with bpf_local_storage_unpin_owner() to make sure + * destroy() sees the correct local_storage->mem_charge. */ smp_mb(); } From e8b084646b946cc6f7976365588035947a296184 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:13 +0000 Subject: [PATCH 566/660] bpf: Validate the SRH which End.DT6 uncovers in End.BPF input_action_end_bpf() validates the SRH of a packet and pulls it into the head before the program runs. If the program decapsulates the packet with bpf_lwt_seg6_action(SEG6_LOCAL_ACTION_END_DT6), bpf_update_srh_state() looks for a routing header in the inner packet and takes it for the new SRH as it is. Nobody has looked at that header, and whoever has sent the packet has written it. ipv6_find_hdr() finds it in a fragment of the skb as well, but hdrlen is then read at that offset from skb->data, past the end of the head. The header need not be an SRH, and its hdrlen need not fit into the packet, but it is marked as valid, and bpf_lwt_seg6_adjust_srh() and bpf_lwt_seg6_store_bytes() check their offsets against it. A program which then adds a TLV at the end of the SRH, where hdrlen has it, makes bpf_lwt_seg6_adjust_srh() move and write memory behind the head of the skb, as far as the sender likes within the 2 KB that hdrlen can say: BUG: KASAN: slab-out-of-bounds in bpf_skb_net_hdr_push+0x185/0x240 Read of size 2088 at addr ffff888107d6f09e by task srh/102 BUG: KASAN: slab-out-of-bounds in bpf_skb_net_hdr_push+0x1b5/0x240 Write of size 8 at addr ffff888107d6f8be by task srh/102 Use seg6_get_srh(), which pulls and validates the header. Without a valid SRH the state says that there is none, and the packet goes its way as before. The pull can move the head, so compute the data pointers behind it. Fixes: 486cdf21583e ("bpf: add End.DT6 action to bpf_lwt_seg6_action helper") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-12-alexei.starovoitov@gmail.com --- net/core/filter.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index 8167cbadc625..1f34cf0f598c 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7025,15 +7025,15 @@ static void bpf_update_srh_state(struct sk_buff *skb) { struct seg6_bpf_srh_state *srh_state = this_cpu_ptr(&seg6_bpf_srh_states); - int srhoff = 0; - if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) { - srh_state->srh = NULL; - } else { - srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); + /* Behind a decapsulation this is a header that nobody has looked at */ + srh_state->srh = seg6_get_srh(skb, 0); + if (srh_state->srh) { srh_state->hdrlen = srh_state->srh->hdrlen << 3; srh_state->valid = true; } + /* it may have been pulled into the head */ + bpf_compute_data_pointers(skb); } BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, From 434324c611a98bc76dc850d45d67756dae5622cf Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:14 +0000 Subject: [PATCH 567/660] bpf: Pull the inner header for End.DT6 in End.BPF bpf_lwt_seg6_action(SEG6_LOCAL_ACTION_END_DT6) pulls the outer headers off the packet and hands it to seg6_lookup_nexthop(), which reads the inner IPv6 header at skb->data. Nothing says that this header is in the head of the skb, or that there is one. If the head ends with the outer headers, the route is looked up for whatever lies behind the head, and the packet is dropped or leaves by the wrong route. If the program then pushes an SRH with SEG6_LOCAL_ACTION_END_B6, seg6_do_srh_inline() pulls an IPv6 header off a head which has none, and __skb_pull() hits its BUG(): the kernel panics on a packet from the network. GRO pulls the inner header of a plain IPv6 in IPv6 packet into the head. It takes a packet which GRO does not look into that far, one with a fragment header in front of the inner header for example, or a device without GRO, and a driver which leaves the inner header in a fragment. Make sure of the inner header together with the outer ones. The program goes on behind an error of the helper, so do it before the packet is changed. A pull which fails can still have moved the head, so look for the SRH again. Fixes: 486cdf21583e ("bpf: add End.DT6 action to bpf_lwt_seg6_action helper") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-13-alexei.starovoitov@gmail.com --- net/core/filter.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/net/core/filter.c b/net/core/filter.c index 1f34cf0f598c..da1f40de03ea 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7066,9 +7066,14 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0) return -EBADMSG; - if (!pskb_pull(skb, hdroff)) + /* As long as nothing has been changed, the program goes on */ + if (!pskb_may_pull(skb, hdroff + sizeof(struct ipv6hdr))) { + /* but the head may have moved */ + bpf_update_srh_state(skb); return -EBADMSG; + } + __skb_pull(skb, hdroff); skb_postpull_rcsum(skb, skb_network_header(skb), hdroff); skb_reset_network_header(skb); skb_reset_transport_header(skb); From 427e751a231aa781441d51c42e716905ee348099 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:15 +0000 Subject: [PATCH 568/660] bpf: Update the SRH state if End.B6 fails late For SEG6_LOCAL_ACTION_END_B6 and END_B6_ENCAP bpf_lwt_seg6_action() looks for the SRH again only if bpf_push_seg6_encap() has succeeded. That function pushes the new header, for which the head of the skb is reallocated if it has not got the room, and then looks up the route to the first segment. If there is none, it returns an error, and the pointer to the SRH in seg6_bpf_srh_states still points into the old head, which has been freed. The program goes on behind the error. bpf_lwt_seg6_store_bytes() and bpf_lwt_seg6_adjust_srh() read the SRH through that pointer. Look for the SRH whatever bpf_push_seg6_encap() says. Fixes: 486cdf21583e ("bpf: add End.DT6 action to bpf_lwt_seg6_action helper") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-14-alexei.starovoitov@gmail.com --- net/core/filter.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index da1f40de03ea..caa9c3ef0a78 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7087,8 +7087,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; case SEG6_LOCAL_ACTION_END_B6_ENCAP: @@ -7096,8 +7096,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; default: From 22f100bd013458836c3d0eaf8050b854d7534a66 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 16:37:16 +0000 Subject: [PATCH 569/660] bpf: Clear sk_refcnt of a clone before its storage is linked sk_clone() sets sk_refcnt of the new socket some lines behind bpf_sk_storage_clone(). But the elements which that function copies are linked to their maps at once, and an element leads to its owner: the iterator over a sk_storage map hands the owner to its program. bpf_sk_storage_get() with BPF_SK_STORAGE_GET_F_CREATE and bpf_sk_storage_delete() rely on refcount_inc_not_zero() to tell whether they may touch the socket. sk_refcnt is in the part of struct sock which sock_copy() leaves out, and sk_prot_alloc() does not clear a clone unless init_on_alloc is on. It is zero in an object which has been a socket before, and anything in one which has not. If it is not zero, a helper gets a reference which refcount_set() wipes out, and the helper's sock_put() then drops one of the two references of the new socket, which is freed too early: IPv4: Attempt to release TCP socket in state 1 BUG: KASAN: slab-use-after-free in __raw_spin_lock_init+0x47/0x90 In the meantime the helper changes a storage which bpf_sk_storage_clone() fills without taking its lock. Nothing else can find the clone at this point. Say that it is not alive yet, before the first element is linked. Sockets whose parent has no storage are left alone. Fixes: a50a85e40c59 ("bpf: Expose bpf_sk_storage_* to iterator programs") Suggested-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-15-alexei.starovoitov@gmail.com --- net/core/bpf_sk_storage.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 1d295a8769fa..b5349bec5695 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -164,6 +164,13 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) if (!sk_storage || hlist_empty(&sk_storage->list)) goto out; + /* + * The maps lead to newsk as soon as an element is linked, but + * sk_clone() sets sk_refcnt only later. Until then it holds whatever + * the slab object held before: no reference can be had yet. + */ + refcount_set(&newsk->sk_refcnt, 0); + hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { struct bpf_local_storage_elem *copy_selem; struct bpf_local_storage_map *smap; From 6186c9711ac6fb984f6c2093c52bf4e827cc50b1 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:17 +0000 Subject: [PATCH 570/660] bpf: Fix NULL deref in the sk_storage map iterator bpf_sk_storage_map_seq_find_next() hands out an element only if its local_storage pointer is set. __bpf_sk_storage_map_seq_show() then reads the pointer a second time and dereferences it without a test. Since bpf_local_storage_destroy() unlinks the elements with bpf_selem_unlink_nofail(), the pointer is reset to NULL when the socket which owns the element is destroyed, so it can change between the two reads. Reading the iterator while sockets with an element in the map are closed ends in a NULL pointer dereference. The sockets need not belong to the reader: where a program gives every new socket an element, an unprivileged user who opens and closes sockets is enough. Skip such an element, as the TCP and UDP iterators skip a socket which has been unhashed. Fixes: 0be08389c7f2 ("bpf: Switch to bpf_selem_unlink_nofail in bpf_local_storage_{map_free, destroy}") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-16-alexei.starovoitov@gmail.com --- net/core/bpf_sk_storage.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index b5349bec5695..aea67b446c7c 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -813,6 +813,9 @@ static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, ctx.map = info->map; if (selem) { sk_storage = rcu_dereference(selem->local_storage); + /* reset since ->next() looked: the socket goes away */ + if (!sk_storage) + return SEQ_SKIP; ctx.sk = sk_storage->owner; ctx.value = SDATA(selem)->data; } From bd897919e4dc18bbfdfc58b696fb948b8bc2c79a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 16:37:18 +0000 Subject: [PATCH 571/660] bpf: Link the first element after its storage is published bpf_local_storage_alloc() links the first element of a new storage to the bucket of its map, and then publishes the storage in the owner with cmpxchg(). If another cpu has been faster, the element is taken out of the bucket again and is freed together with the storage. In between the iterator over a sk_storage map can find the element. It stays linked to a storage that points to the socket, but the socket knows nothing of that storage: bpf_sk_storage_free() neither unlinks the element nor waits for anybody who uses the storage. So nothing tells the iterator that the socket has gone away. Three cpus that create the first storage of a new AF_UNIX socket at the same time, and a fourth that reads the iterator, are enough: in 20 seconds its program was shown some 4000 elements of storages that had lost. If the socket is closed in time, the program is handed a freed socket. Link the element once the storage is published. Both happens under the lock of the bucket, which holds off whoever finds the element through the owner and wants to unlink it. Fixes: c69d2ddb2072 ("bpf: Using rcu_read_lock for bpf_sk_storage_map iterator") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-17-alexei.starovoitov@gmail.com --- kernel/bpf/bpf_local_storage.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 7721834e4bff..c08afb33d038 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -509,8 +509,6 @@ int bpf_local_storage_alloc(void *owner, if (err) goto uncharge; - bpf_selem_link_map_nolock(b, first_selem); - owner_storage_ptr = (struct bpf_local_storage **)owner_storage(smap, owner); /* Publish storage to the owner. @@ -525,11 +523,17 @@ int bpf_local_storage_alloc(void *owner, */ prev_storage = cmpxchg(owner_storage_ptr, NULL, storage); if (unlikely(prev_storage)) { - bpf_selem_unlink_map_nolock(first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); err = -EAGAIN; goto uncharge; } + /* + * Not before the storage is published: the owner does not wait for + * users of a storage that it knows nothing of, so that nobody may find + * such a storage through the map. b->lock holds off those who find + * first_selem through the owner and want to unlink it. + */ + bpf_selem_link_map_nolock(b, first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); return 0; From 070a96b2a98f66087797d8a31d53126de7a5208f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:19 +0000 Subject: [PATCH 572/660] bpf: Hold the socket in the sk_storage map iterator The iterator over a sk_storage map passes the owner of an element to its program as ctx->sk, without a reference. All that it holds is the RCU read lock, which keeps the element and the storage, not the socket. An element is unlinked by bpf_sk_storage_free(), which __sk_destruct() calls right before it frees the socket. That is no better for a socket with SOCK_RCU_FREE: the grace period comes in front of __sk_destruct(), so a reader which starts behind call_rcu() still finds the element. The program and the helpers it hands ctx->sk to, such as bpf_sk_storage_get() or bpf_sk_storage_delete(), use a socket which is being freed on another CPU. It takes nothing but reading the iterator while sockets with an element in the map are closed, by whomever. Take a reference for the time of the program, and skip the element if there is none to be had. To look at the owner at all, do what bpf_selem_unlink_nofail() does for bpf_local_storage_map_free(): as long as the element is linked under the lock of the storage, bpf_local_storage_destroy() has not got past it, unless it has given up on the lock, and then it waits for owner_refcnt. The last reference may now be dropped by the iterator. The end of a socket may want process context: rfcomm_sock_destruct() takes a mutex. So only drop the reference under RCU, and free a socket whose last one it was when the RCU read lock has been released, which is at the end of a bucket and of every read() at the latest. Fixes: c69d2ddb2072 ("bpf: Using rcu_read_lock for bpf_sk_storage_map iterator") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-18-alexei.starovoitov@gmail.com --- net/core/bpf_sk_storage.c | 73 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index aea67b446c7c..ea22ec5ef5b1 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -699,8 +699,37 @@ struct bpf_iter_seq_sk_storage_map_info { struct bpf_map *map; unsigned int bucket_id; unsigned skip_elems; + struct rcu_head *dead_sks; }; +/* + * sock_put() in two halves. The end of a socket may want process context: + * rfcomm_sock_destruct() takes a mutex. If the iterator drops the last + * reference, it keeps the socket until it has left the RCU read side critical + * section. Nobody uses sk_rcu before sk_destruct(). + */ +static void +bpf_sk_storage_map_seq_put_sk(struct bpf_iter_seq_sk_storage_map_info *info, + struct sock *sk) +{ + if (refcount_dec_and_test(&sk->sk_refcnt)) { + sk->sk_rcu.next = info->dead_sks; + info->dead_sks = &sk->sk_rcu; + } +} + +static void +bpf_sk_storage_map_seq_free_sks(struct bpf_iter_seq_sk_storage_map_info *info) +{ + struct sock *sk; + + while (info->dead_sks) { + sk = container_of(info->dead_sks, struct sock, sk_rcu); + info->dead_sks = sk->sk_rcu.next; + sk_free(sk); + } +} + static struct bpf_local_storage_elem * bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, struct bpf_local_storage_elem *prev_selem) @@ -729,6 +758,7 @@ bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, /* not found, unlock and go to the next bucket */ b = &smap->buckets[bucket_id++]; rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(info); skip_elems = 0; break; } @@ -796,12 +826,44 @@ DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, struct bpf_map *map, struct sock *sk, void *value) +/* + * Take a reference on the socket which owns @selem. RCU keeps the element, not + * its owner. As long as the element is linked under the lock of the storage, + * the owner has not got past bpf_sk_storage_free(), unless that has given up + * on the lock. Then it still waits for owner_refcnt. + */ +static struct sock * +bpf_sk_storage_selem_get_sk(struct bpf_local_storage_elem *selem) +{ + struct bpf_local_storage *sk_storage; + struct sock *sk = NULL; + unsigned long flags; + + sk_storage = rcu_dereference(selem->local_storage); + /* reset since ->next() looked: the socket goes away */ + if (!sk_storage) + return NULL; + + if (raw_res_spin_lock_irqsave(&sk_storage->lock, flags)) + return NULL; + + if (!hlist_unhashed(&selem->snode) && + refcount_inc_not_zero(&sk_storage->owner_refcnt)) { + sk = sk_storage->owner; + if (!refcount_inc_not_zero(&sk->sk_refcnt)) + sk = NULL; + bpf_local_storage_unpin_owner(sk_storage); + } + raw_res_spin_unlock_irqrestore(&sk_storage->lock, flags); + + return sk; +} + static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, struct bpf_local_storage_elem *selem) { struct bpf_iter_seq_sk_storage_map_info *info = seq->private; struct bpf_iter__bpf_sk_storage_map ctx = {}; - struct bpf_local_storage *sk_storage; struct bpf_iter_meta meta; struct bpf_prog *prog; int ret = 0; @@ -812,14 +874,14 @@ static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, ctx.meta = &meta; ctx.map = info->map; if (selem) { - sk_storage = rcu_dereference(selem->local_storage); - /* reset since ->next() looked: the socket goes away */ - if (!sk_storage) + ctx.sk = bpf_sk_storage_selem_get_sk(selem); + if (!ctx.sk) return SEQ_SKIP; - ctx.sk = sk_storage->owner; ctx.value = SDATA(selem)->data; } ret = bpf_iter_run_prog(prog, &ctx); + if (ctx.sk) + bpf_sk_storage_map_seq_put_sk(info, ctx.sk); } return ret; @@ -837,6 +899,7 @@ static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) (void)__bpf_sk_storage_map_seq_show(seq, v); else rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(seq->private); } static int bpf_iter_init_sk_storage_map(void *priv_data, From 13ac7c05bc45d3445b580da186548dd3487fae4e Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:20 +0000 Subject: [PATCH 573/660] bpf: Bound the nesting that the BTF show code follows btf_struct_show() and btf_array_show() recurse for every nested struct and array. MAX_RESOLVE_DEPTH bounds one resolve walk when BTF is loaded, not the nesting: a member whose type is resolved already is not walked again, so BTF that defines its types from the inside out can nest them as deep as it has types. Whoever may load BTF, with CAP_BPF or a BPF token, can give such a type to the value of a map. Reading the file of that map in bpffs then overruns the kernel stack: with a few hundred levels the stack guard page is hit and the kernel panics. Stop at 32 levels. That is MAX_RESOLVE_DEPTH, and __btf_show_indent() has as many columns. Nothing is shown for a member that lies deeper, and the status becomes -E2BIG, which bpf_snprintf_btf() and bpf_seq_printf_btf() return. The types of an x86_64 defconfig kernel nest 14 deep. Fixes: b00b8daec828 ("bpf: btf: Add pretty print capability for data with BTF type info") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-19-alexei.starovoitov@gmail.com --- kernel/bpf/btf.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 64c78434fa05..a9bf4c3c0a27 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1081,6 +1081,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, } #define BTF_SHOW_MAX_ITER 10 +/* nested structs and arrays shown; __btf_show_indent() has as many columns */ +#define BTF_SHOW_MAX_DEPTH 32 #define BTF_KIND_BIT(kind) (1ULL << kind) @@ -1474,8 +1476,18 @@ static void *btf_show_start_aggr_type(struct btf_show *show, const struct btf_type *t, u32 type_id, void *data) { - void *safe_data = btf_show_start_type(show, t, type_id, data); + void *safe_data; + /* + * Structs and arrays are shown by recursion. BTF loading bounds the + * depth of one resolve walk, not how deep resolved types may nest. + */ + if (show->state.depth >= BTF_SHOW_MAX_DEPTH) { + show->state.status = -E2BIG; + return NULL; + } + + safe_data = btf_show_start_type(show, t, type_id, data); if (!safe_data) return safe_data; From 45e912fb1db00404267fafd9fd034c68b1f94622 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:21 +0000 Subject: [PATCH 574/660] bpf: Check the new program of an iterator link update The bpf_map_elem, bpf_sk_storage_map and sockmap iterators hand the key and the value of an element to the program as buffers. The verifier only records how far a program reaches into them, and ->attach_target() compares that with the key and value size of the map. bpf_iter_link_replace() does not. BPF_LINK_UPDATE installs a program that an attach refuses with -EACCES, and the program then reads, and for a value writes, behind every element that the iterator visits. It takes CAP_BPF and CAP_PERFMON, or a BPF token. Split the test out of the three ->attach_target() and ask it, as ->check_prog(), for the new program of an update as well. Targets without such a test take every program of their kind, as before. Fixes: d6c4503cc296 ("bpf: Implement bpf iterator for hash maps") Fixes: 5ce6e77c7edf ("bpf: Implement bpf iterator for sock local storage map") Fixes: 0365351524d7 ("net: Allow iterating sockmap and sockhash") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-20-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 4 +++ kernel/bpf/bpf_iter.c | 7 ++++++ kernel/bpf/map_iter.c | 53 ++++++++++++++++++++++----------------- net/core/bpf_sk_storage.c | 19 ++++++++++---- net/core/sock_map.c | 19 ++++++++++---- 5 files changed, 69 insertions(+), 33 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 46d325aa178e..3fb3c7d22d99 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3128,6 +3128,8 @@ typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); +typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, @@ -3146,6 +3148,8 @@ struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; + /* may the program run on what the link is attached to? */ + bpf_iter_check_prog_t check_prog; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index fc3a75684ef7..4668c1696a08 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -436,6 +436,13 @@ static int bpf_iter_link_replace(struct bpf_link *link, goto out_unlock; } + if (iter_link->tinfo->reg_info->check_prog) { + ret = iter_link->tinfo->reg_info->check_prog(new_prog, + &iter_link->aux); + if (ret) + goto out_unlock; + } + old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 2077d46d167c..16537356c1a2 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -97,13 +97,29 @@ static struct bpf_iter_reg bpf_map_reg_info = { .seq_info = &bpf_map_seq_info, }; +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + const struct bpf_map *map = aux->map; + u32 value_size = map->value_size; + + if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + value_size = round_up(value_size, 8) * num_possible_cpus(); + + if (prog->aux->max_rdonly_access > map->key_size || + prog->aux->max_rdwr_access > value_size) + return -EACCES; + + return 0; +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) { - u32 key_acc_size, value_acc_size, key_size, value_size; struct bpf_map *map; - bool is_percpu = false; int err = -EINVAL; if (!linfo->map.map_fd) @@ -123,30 +139,20 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, goto put_map; } - if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) - is_percpu = true; - else if (map->map_type != BPF_MAP_TYPE_HASH && - map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY && - map->map_type != BPF_MAP_TYPE_RHASH) + if (map->map_type != BPF_MAP_TYPE_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && + map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_LRU_HASH && + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_RHASH) goto put_map; - key_acc_size = prog->aux->max_rdonly_access; - value_acc_size = prog->aux->max_rdwr_access; - key_size = map->key_size; - if (!is_percpu) - value_size = map->value_size; - else - value_size = round_up(map->value_size, 8) * num_possible_cpus(); - - if (key_acc_size > key_size || value_acc_size > value_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -179,6 +185,7 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .target = "bpf_map_elem", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index ea22ec5ef5b1..72b9ee4c98de 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -919,6 +919,15 @@ static void bpf_iter_fini_sk_storage_map(void *priv_data) bpf_map_put_with_uref(seq_info->map); } +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdwr_access > aux->map->value_size) + return -EACCES; + + return 0; +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -936,12 +945,11 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) goto put_map; - if (prog->aux->max_rdwr_access > map->value_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -972,6 +980,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { .target = "bpf_sk_storage_map", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 38df84284328..057e2f3c0265 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1933,6 +1933,15 @@ out: return ret; } +static int sock_map_iter_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdonly_access > aux->map->key_size) + return -EACCES; + + return 0; +} + static int sock_map_iter_attach_target(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -1951,12 +1960,11 @@ static int sock_map_iter_attach_target(struct bpf_prog *prog, map->map_type != BPF_MAP_TYPE_SOCKHASH) goto put_map; - if (prog->aux->max_rdonly_access > map->key_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = sock_map_iter_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -1973,6 +1981,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { .target = "sockmap", .attach_target = sock_map_iter_attach_target, .detach_target = sock_map_iter_detach_target, + .check_prog = sock_map_iter_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, From 8e83d53596445fcf5b0385e5fbe8bcd971e06d06 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:22 +0000 Subject: [PATCH 575/660] bpf: Fix stale offload->prog pointer after realloc bpf_prog_realloc() moves a program that the rewrites of the verifier make outgrow its pages. It updates aux->prog, but not the offload->prog of a device-bound program. When the device is unregistered, __bpf_offload_dev_netdev_unregister() hands the stale pointer to __bpf_prog_offload_destroy(), which faults on the freed program: BUG: unable to handle page fault for address: ffffc90000071050 RIP: 0010:__bpf_prog_offload_destroy+0x3f/0x310 It takes the right to load a device-bound XDP program, CAP_BPF and CAP_NET_ADMIN or a BPF token, and a device that can be removed. Commit a1aa9ef47c29 ("bpf: Fix stale offload->prog pointer after constant blinding") repaired the other place that replaces a program. Do the same here. Nothing keeps the device while a program is loaded, so it can also be unregistered at that very moment. That frees the offload, and it may have read offload->prog already when the old copy of the program is freed. In both places update the pointer under bpf_devs_lock, before the old copy goes. Fixes: ab3f0063c48c ("bpf: offload: add infrastructure for loading programs for a specific netdev") Fixes: 2b3486bc2d23 ("bpf: Introduce device-bound XDP programs") Fixes: a1aa9ef47c29 ("bpf: Fix stale offload->prog pointer after constant blinding") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-21-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 5 +++++ kernel/bpf/core.c | 4 ++-- kernel/bpf/offload.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 3fb3c7d22d99..0de5947dceaa 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3792,6 +3792,7 @@ int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); +void bpf_prog_dev_bound_moved(struct bpf_prog *prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) @@ -3853,6 +3854,10 @@ static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, return -EOPNOTSUPP; } +static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ +} + static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 642f0ab8eb2d..69285abcf594 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -269,6 +269,7 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); fp->pages = pages; fp->aux->prog = fp; + bpf_prog_dev_bound_moved(fp); /* We keep fp->aux from fp_old around in the new * reallocated structure. @@ -1558,8 +1559,7 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) * know whether fp here is the clone or the original. */ fp->aux->prog = fp; - if (fp->aux->offload) - fp->aux->offload->prog = fp; + bpf_prog_dev_bound_moved(fp); bpf_prog_clone_free(fp_other); } diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index d855399812ee..5a10f40a7831 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -96,6 +96,22 @@ err_free: return err; } +/* + * @prog takes the place of a copy that the caller is about to free. Nothing + * keeps the netdev while a program is loaded: once the lock is dropped, no + * unregistration is left that has got the old copy from offload->prog. + */ +void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ + if (!bpf_prog_is_dev_bound(prog->aux)) + return; + + down_read(&bpf_devs_lock); + if (prog->aux->offload) + prog->aux->offload->prog = prog; + up_read(&bpf_devs_lock); +} + static void __bpf_prog_offload_destroy(struct bpf_prog *prog) { struct bpf_prog_offload *offload = prog->aux->offload; From af929bc938e0d0ecd839e6fed9a2ce5eb5185e5c Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:23 +0000 Subject: [PATCH 576/660] bpf: Don't bind a program to a netdev that goes away bpf_prog_dev_bound_init() looks the device up, takes bpf_devs_lock and registers the device for the program. The reference it holds does not keep the device registered. If bpf_dev_bound_netdev_unregister() has run by the time the lock is taken, nobody takes the program off the device any more: the bpf_offload_netdev and offload->netdev point to a net_device that is freed as soon as the reference is dropped. bpf_dev_bound_resolve_kfunc() dereferences it while the program is verified, and a new device at the same address finds the old entry. unregister_netdevice_many_notify() leaves NETREG_REGISTERED before it calls bpf_dev_bound_netdev_unregister(), which takes bpf_devs_lock. Look at reg_state under that lock, and treat a device that goes like one that is not there. Fixes: 2b3486bc2d23 ("bpf: Introduce device-bound XDP programs") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-22-alexei.starovoitov@gmail.com --- kernel/bpf/offload.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 5a10f40a7831..f7bfff1760cc 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -266,7 +266,14 @@ int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr) prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY); down_write(&bpf_devs_lock); - err = __bpf_prog_dev_bound_init(prog, netdev); + /* + * Behind bpf_dev_bound_netdev_unregister() nobody would take the + * program off the device, which goes once the reference is dropped. + */ + if (READ_ONCE(netdev->reg_state) == NETREG_REGISTERED) + err = __bpf_prog_dev_bound_init(prog, netdev); + else + err = -EINVAL; up_write(&bpf_devs_lock); out: From 1e28a5e52d32b7d3f4c9fb24898774f941166d2b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:24 +0000 Subject: [PATCH 577/660] bpf: Keep a program until its bpf_wq callback has ended bpf_wq_work() runs the callback under rcu_read_lock_trace() only, since the callback may sleep. The program it belongs to need not be sleepable, and such a program is freed behind a plain RCU grace period. The reference that the bpf_wq holds is dropped when the map element goes, or when another callback is set, no matter whether the work is running. If that was the last reference the program is freed under the running callback, and the worker goes on in the freed JIT image: Oops: int3: 0000 [#1] SMP NOPTI RIP: 0010:0xffffffffc0600947 All it takes is to close the map and the program while the callback runs. Note in the program that it has a callback that runs sleepable, and wait for an RCU Tasks Trace grace period, which implies a plain one, before such a program is freed. Let bpf_wq_work() enter its read side critical section before it reads callback_fn. bpf_task_work holds a reference to the program until the callback is done. Fixes: 8e83da9732d9 ("bpf: add bpf_wq_start") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-23-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 10 ++++++++++ kernel/bpf/helpers.c | 9 ++++++--- kernel/bpf/syscall.c | 4 ++-- kernel/bpf/verifier.c | 2 ++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0de5947dceaa..db56d5d07462 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1873,6 +1873,7 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool tramp_linked; /* true if it was ever linked to a trampoline */ + bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; @@ -2006,6 +2007,15 @@ struct bpf_prog { }; }; +/* + * Does the program, or a callback of it, run under RCU Tasks Trace protection + * only, outside of any RCU read side critical section? + */ +static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog) +{ + return prog->sleepable || prog->aux->sleepable_async_cb; +} + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 439370d73cc5..648abe1ba63a 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1262,18 +1262,21 @@ static void bpf_wq_work(struct work_struct *work) BTF_TYPE_EMIT(struct bpf_wq); + /* keeps the program that callback_fn belongs to */ + rcu_read_lock_trace(); + callback_fn = READ_ONCE(cb->callback_fn); if (!callback_fn) - return; + goto out; key = map_key_from_value(map, value, &idx); - rcu_read_lock_trace(); - migrate_disable(); + migrate_disable(); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); migrate_enable(); +out: rcu_read_unlock_trace(); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0f5eb55d49d6..30d570c1c9a0 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2477,7 +2477,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) { struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); else call_rcu(rcu, __bpf_prog_put_rcu); @@ -2497,7 +2497,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) if (deferred) { if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); - else if (prog->sleepable) + else if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d0ba7156ba4f..6b15862a15e8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11211,6 +11211,8 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins is_async_cb_sleepable(env, insn)); if (IS_ERR(async_cb)) return PTR_ERR(async_cb); + if (is_async_cb_sleepable(env, insn)) + env->prog->aux->sleepable_async_cb = true; callee = async_cb->frame[0]; callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; From 927c9c985f3ef4d30c086884b7e0ecb933f742ef Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:25 +0000 Subject: [PATCH 578/660] bpf: Count sleepable callbacks as sleepable map users map->sleepable_refcnt tells bpf_map_fd_put_ptr() whether an inner map that is taken out of its outer map has to wait for an RCU Tasks Trace grace period as well. It counts the sleepable programs that use the outer map. The bpf_wq or bpf_task_work callback of a program that is not sleepable runs under rcu_read_lock_trace() only, and is not counted. Such a callback can look an inner map up and keep it while user space deletes it from the outer map. The inner map is freed behind a plain RCU grace period, under the callback: BUG: KASAN: slab-use-after-free in array_map_update_elem+0x6bc/0x7d0 Count a program with such a callback. The maps are bound before the verifier meets the callback, so raise the count of those that are bound when it does. Fixes: 8e83da9732d9 ("bpf: add bpf_wq_start") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-24-alexei.starovoitov@gmail.com --- kernel/bpf/core.c | 2 +- kernel/bpf/syscall.c | 2 +- kernel/bpf/verifier.c | 17 +++++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 69285abcf594..e1216927373a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3042,7 +3042,7 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, bool sleepable; u32 i; - sleepable = aux->prog->sleepable; + sleepable = bpf_prog_runs_sleepable(aux->prog); for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 30d570c1c9a0..c412bcd1903f 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6257,7 +6257,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) /* The bpf program will not access the bpf map, but for the sake of * simplicity, increase sleepable_refcnt for sleepable program as well. */ - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6b15862a15e8..1f2a846ae415 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -593,6 +593,19 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn return false; } +/* From here on the maps of the program count it as a sleepable user. */ +static void mark_sleepable_async_cb(struct bpf_verifier_env *env) +{ + int i; + + if (bpf_prog_runs_sleepable(env->prog)) + return; + + env->prog->aux->sleepable_async_cb = true; + for (i = 0; i < env->used_map_cnt; i++) + atomic64_inc(&env->used_maps[i]->sleepable_refcnt); +} + bool bpf_is_may_goto_insn(struct bpf_insn *insn) { return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; @@ -11212,7 +11225,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins if (IS_ERR(async_cb)) return PTR_ERR(async_cb); if (is_async_cb_sleepable(env, insn)) - env->prog->aux->sleepable_async_cb = true; + mark_sleepable_async_cb(env); callee = async_cb->frame[0]; callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; @@ -20776,7 +20789,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (err) return err; - if (env->prog->sleepable) + if (bpf_prog_runs_sleepable(env->prog)) atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, From 3e6d7522af2cd1b8bf135ee7df6ca962986f44d6 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:26 +0000 Subject: [PATCH 579/660] bpf: Check the maps that sleepable callbacks use Sleepable programs can only use the kinds of map that do without an RCU read side critical section. check_map_prog_compatibility() sees to that when a map is bound to the program, and asks whether the program is sleepable. That misses the bpf_wq or bpf_task_work callback of a program that is not, which runs sleepable all the same, and it misses an inner map, which is not bound. The bpf_wq callback of a tc program that looks a key up in a sockhash gets as far as WARNING: net/core/sock_map.c:903 at __sock_hash_lookup_elem+0x18f/0x1e0 Ask the same question for the map that is handed to a helper in sleepable context, unless the RCU read lock is held: by bpf_rcu_read_lock(), or by the helper that runs the callback, as bpf_for_each_map_elem() does for a hash map. Programs that take the lock go on working. Disabled preemption or interrupts are not enough, since the maps ask for rcu_read_lock_held(). The other parts of the program may use such a map as before. A global function that is called with preemption or interrupts disabled is verified for the non-sleepable context, in which nothing is known of all that. Verify it for a context of its own: not sleepable, and without the RCU read lock. It differs in what maps can be used, and in nothing else. What is refused from now on runs into the warning above, or uses what the map frees behind a plain RCU grace period: an entry of a devmap, of a perf event array or of a cgroup array. A bloom filter does without RCU. Add it to the list, so that callbacks which use one go on loading. As a side effect sleepable programs can use bloom filters as well. A pointer to the value of a devmap which a program keeps beyond bpf_rcu_read_unlock() is not dealt with here. Fixes: 638e4b825d52 ("bpf: Allows per-cpu maps and map-in-map in sleepable programs") Fixes: 8e83da9732d9 ("bpf: add bpf_wq_start") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-25-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 14 ++++- include/linux/bpf_verifier.h | 2 + kernel/bpf/verifier.c | 109 ++++++++++++++++++++++++----------- 3 files changed, 89 insertions(+), 36 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index db56d5d07462..1823ca3a1d5e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1742,12 +1742,20 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif +/* The contexts that a global function is called in, and is verified for. */ +enum bpf_func_context { + BPF_FUNC_CTX_NON_SLEEPABLE, + BPF_FUNC_CTX_SLEEPABLE, + /* Sleepable code has disabled preemption or interrupts: no RCU read lock. */ + BPF_FUNC_CTX_NO_RCU_LOCK, + BPF_FUNC_CTX_CNT, +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; - /* Indexed by in_sleepable. */ - bool called[2]; - bool verified[2]; + bool called[BPF_FUNC_CTX_CNT]; + bool verified[BPF_FUNC_CTX_CNT]; }; enum bpf_jit_poke_reason { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 816948df8fff..e420d211f423 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1090,6 +1090,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1f2a846ae415..81660ffce99a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -219,6 +219,7 @@ static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); static inline bool in_sleepable_context(struct bpf_verifier_env *env); +static bool in_rcu_callback(struct bpf_verifier_env *env); static const char *non_sleepable_context_description(struct bpf_verifier_env *env); static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); @@ -10379,12 +10380,71 @@ bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env) bpf_jit_supports_subprog_tailcalls(); } +/* + * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled + * preemption or interrupts will not do. + */ +static bool has_rcu_read_lock(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + + if (state->active_rcu_locks || in_rcu_callback(env)) + return true; + if (in_sleepable(env)) + return false; + return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn; +} + +/* May the map be used outside of an RCU read side critical section? */ +static bool map_needs_no_rcu(const struct bpf_map *map) +{ + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_PERCPU_ARRAY: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_LPM_TRIE: + case BPF_MAP_TYPE_ARRAY_OF_MAPS: + case BPF_MAP_TYPE_HASH_OF_MAPS: + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + case BPF_MAP_TYPE_INODE_STORAGE: + case BPF_MAP_TYPE_SK_STORAGE: + case BPF_MAP_TYPE_TASK_STORAGE: + case BPF_MAP_TYPE_CGRP_STORAGE: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_ARENA: + case BPF_MAP_TYPE_INSN_ARRAY: + case BPF_MAP_TYPE_PROG_ARRAY: + case BPF_MAP_TYPE_BLOOM_FILTER: + return true; + default: + return false; + } +} + static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id) { if (!map) return 0; + /* + * check_map_prog_compatibility() asks the program. That misses the + * sleepable callback of another program, and an inner map. They may + * use any map under the RCU read lock. Disabled preemption or + * interrupts will not do: the maps ask for rcu_read_lock_held(). + */ + if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) { + verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n", + map->name); + return -EINVAL; + } + /* We need a two way check, first is from map perspective ... */ switch (map->map_type) { case BPF_MAP_TYPE_PROG_ARRAY: @@ -11373,7 +11433,12 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); - sub_aux->called[in_sleepable_context(env)] = true; + if (in_sleepable_context(env)) + sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true; + else if (has_rcu_read_lock(env)) + sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true; + else + sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true; returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); @@ -20676,35 +20741,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EINVAL; } - if (prog->sleepable) - switch (map->map_type) { - case BPF_MAP_TYPE_HASH: - case BPF_MAP_TYPE_RHASH: - case BPF_MAP_TYPE_LRU_HASH: - case BPF_MAP_TYPE_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_LPM_TRIE: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_RINGBUF: - case BPF_MAP_TYPE_USER_RINGBUF: - case BPF_MAP_TYPE_INODE_STORAGE: - case BPF_MAP_TYPE_SK_STORAGE: - case BPF_MAP_TYPE_TASK_STORAGE: - case BPF_MAP_TYPE_CGRP_STORAGE: - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_ARENA: - case BPF_MAP_TYPE_INSN_ARRAY: - case BPF_MAP_TYPE_PROG_ARRAY: - break; - default: - verbose(env, - "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); - return -EINVAL; - } + if (prog->sleepable && !map_needs_no_rcu(map)) { + verbose(env, + "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); + return -EINVAL; + } if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { @@ -21426,7 +21467,8 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) +static int do_check_common(struct bpf_verifier_env *env, int subprog, + enum bpf_func_context context) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); @@ -21446,7 +21488,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = is_sleepable; + state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE; + env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -21663,7 +21706,7 @@ again: /* * We can't loop forever as each pass verifies at least one new context, - * and there are only two contexts per global subprog. + * and there are only three contexts per global subprog. */ if (new_cnt) goto again; From 7fa323d77688e8b62b2f0bf6c3208a2c96d4a5e7 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:27 +0000 Subject: [PATCH 580/660] bpf: Assume no bounds for buffers behind a tail call The bpf_map_elem, bpf_sk_storage_map and sockmap iterators hand the key and the value of an element to the program as buffers. The verifier notes how far a program reaches into them, and the iterator compares that with the map when the program is attached. A program that does not touch the buffers passes. It can tail-call another program of the same target, which gets the same context, and whose reach nobody compares with the map: BUG: KASAN: slab-out-of-bounds in ___bpf_prog_run+0xa46a/0xae80 The program array can be filled at any time, so there is nothing to look at when the link is made. For the same reason a program with a tail call is taken to use the whole stack and the whole packet. Take it to reach as far as can be into the buffers as well, so that these iterators refuse it with -EACCES. Fixes: d6c4503cc296 ("bpf: Implement bpf iterator for hash maps") Fixes: 5ce6e77c7edf ("bpf: Implement bpf iterator for sock local storage map") Fixes: 0365351524d7 ("net: Allow iterating sockmap and sockhash") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-26-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 608621d7f225..002c3935a8eb 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2151,6 +2151,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (!bpf_allow_tail_call_in_subprogs(env)) prog->aux->stack_depth = MAX_BPF_STACK; prog->aux->max_pkt_offset = MAX_PACKET_OFF; + prog->aux->max_rdonly_access = U32_MAX; + prog->aux->max_rdwr_access = U32_MAX; /* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal From 7cc639dc45c5d3b69acb2a2fc834e9eec51e0214 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:28 +0000 Subject: [PATCH 581/660] bpf: Don't take an extension for an iterator program An extension inherits the expected_attach_type of the program it extends. If that is BPF_TRACE_ITER, check_attach_btf_id() hands the extension to bpf_iter_prog_supported(), which goes by the name of the attach function. For an extension that is the name of the subprogram it replaces, so the load fails with -EINVAL, unless that subprogram is called like the function of a target, bpf_iter_bpf_map_elem for example. Then the extension gets the context of that target. If the subprogram has as many arguments, it can use the key and the value as buffers, but how far it reaches is noted in the extension, where the test of the iterator link does not look: it reads and writes behind the elements of the map. And the id of the subprogram in the BTF of the program is cached as the id of the target, so that programs of that target which are loaded already cannot be attached. Refuse all extensions here. Fixes: 15d83c4d7cef ("bpf: Allow loading of a bpf_iter program") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-27-alexei.starovoitov@gmail.com --- kernel/bpf/bpf_iter.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 4668c1696a08..5fd376bd210b 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -351,6 +351,13 @@ int bpf_iter_prog_supported(struct bpf_prog *prog) const char *prefix = BPF_ITER_FUNC_PREFIX; int prefix_len = strlen(prefix); + /* + * The attach function of an extension is a subprogram of the iterator + * program that it extends, whatever its name is. + */ + if (prog->type == BPF_PROG_TYPE_EXT) + return -EINVAL; + if (strncmp(attach_fname, prefix, prefix_len)) return -EINVAL; From cd0490e671d70326d70c7d36e82388939fd9b071 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:29 +0000 Subject: [PATCH 582/660] bpf: Assume no bounds for a writable tp with tail calls A writable raw tracepoint hands a buffer to the program. The verifier notes how far the program reaches into it, and bpf_probe_register() refuses a program that reaches further than the tracepoint allows. A program that does not touch the buffer passes, and can tail-call one of its type that reads and writes behind it. Take a BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE program with a tail call to reach as far as can be, as for the stack, the packet and the buffers of iterators. It cannot be attached any more. Plain raw tracepoint programs, for which the limit is zero, cannot get at the buffer, and a tail call stays within one type: they are left alone. Fixes: 9df1c28bb752 ("bpf: add writable context for raw tracepoints") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-28-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 002c3935a8eb..fe59e3114d82 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2153,6 +2153,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) prog->aux->max_pkt_offset = MAX_PACKET_OFF; prog->aux->max_rdonly_access = U32_MAX; prog->aux->max_rdwr_access = U32_MAX; + if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) + prog->aux->max_tp_access = U32_MAX; /* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal From 7fe348f18b37b3f4c8beb981ef3f84564047380a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 16:37:30 +0000 Subject: [PATCH 583/660] bpf: Don't take an extension for a writable raw tracepoint program bpf_probe_register() compares max_tp_access of the program that is attached with the size of the buffer that the tracepoint lets programs write to. An extension of that program, or of one of its global functions, gets the same context and the same pointer to the buffer. What it accesses is noted in its own max_tp_access, which nobody looks at. So it reads and writes behind the buffer, of a tracepoint that may have none: BUG: KASAN: stack-out-of-bounds in bpf_prog_5843f2b820eb4929_tr26_ext_far+0xf4/0x10c Write of size 4 at addr ffffc90000d8f894 by task wext/103 Refuse such extensions. Fixes: be8704ff07d2 ("bpf: Introduce dynamic program extensions") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-29-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 81660ffce99a..0d1b729989d6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22232,6 +22232,15 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension programs should be JITed\n"); return -EINVAL; } + /* + * How much of the buffer of the tracepoint is used + * is checked for the program that is attached to it. + */ + if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { + bpf_log(log, + "Cannot extend a writable raw tracepoint program\n"); + return -EINVAL; + } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; From 8d0358f6e8b31fcf7d355026a83aac70cc675b19 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:31 +0000 Subject: [PATCH 584/660] bpf: Keep iterator programs off special map fields The bpf_map_elem iterator, for all but the per-cpu maps, and the bpf_sk_storage_map iterator hand the live value of an element to the program as a writable buffer. The verifier does not know the map when it checks the program, so nothing keeps the program from writing over a bpf_spin_lock, a bpf_timer, a kptr or another special field, which neither a map update nor a pointer to a map value can touch. With CAP_BPF and CAP_PERFMON, or a BPF token, a program can so make the kernel follow a pointer of its choice: after a write over a bpf_timer the kernel crashes on the written value when the element is removed. Offsets into a buffer are constants. Let the verifier record the range of bytes that a program may write through a read-write buffer, and refuse the program with -EACCES, at attach and at a link update, if a special field of the map lies in that range. A program that only reads, or that writes beside the special fields, works as before. One that writes on both sides of a special field is refused, since the range covers the field. Fixes: d6c4503cc296 ("bpf: Implement bpf iterator for hash maps") Fixes: 5ce6e77c7edf ("bpf: Implement bpf iterator for sock local storage map") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-30-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 5 +++++ kernel/bpf/map_iter.c | 36 ++++++++++++++++++++++++++++++++---- kernel/bpf/verifier.c | 12 +++++++++--- net/core/bpf_sk_storage.c | 2 +- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1823ca3a1d5e..22947fa192e4 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1866,6 +1866,9 @@ struct bpf_prog_aux { u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; + /* [min, max): the bytes of a rdwr buffer that may be written */ + u32 min_rdwr_write; + u32 max_rdwr_write; u32 subprog_start; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; @@ -3204,6 +3207,8 @@ void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 16537356c1a2..a58c148692e6 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -97,22 +97,50 @@ static struct bpf_iter_reg bpf_map_reg_info = { .seq_info = &bpf_map_seq_info, }; +/* + * For an iterator that hands the live map value to the program as a writable + * buffer: the verifier knows nothing of the map, so the special fields of the + * value have to lie outside of what the program may write. + */ +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map) +{ + const struct btf_record *rec = map->record; + const struct btf_field *field; + u32 i; + + if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + return 0; + + for (i = 0; i < rec->cnt; i++) { + field = &rec->fields[i]; + if (field->offset < prog->aux->max_rdwr_write && + field->offset + field->size > prog->aux->min_rdwr_write) + return -EACCES; + } + + return 0; +} + static int bpf_iter_map_check_prog(const struct bpf_prog *prog, const struct bpf_iter_aux_info *aux) { const struct bpf_map *map = aux->map; u32 value_size = map->value_size; + bool is_percpu; - if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; + if (is_percpu) value_size = round_up(value_size, 8) * num_possible_cpus(); if (prog->aux->max_rdonly_access > map->key_size || prog->aux->max_rdwr_access > value_size) return -EACCES; - return 0; + /* the program gets a copy of a per-cpu value */ + return is_percpu ? 0 : bpf_iter_map_check_value_write(prog, map); } static int bpf_iter_attach_map(struct bpf_prog *prog, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0d1b729989d6..2c812a5f1da8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6053,9 +6053,10 @@ static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, - u32 *max_access) + u32 *max_access, bool write) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; + struct bpf_prog_aux *aux = env->prog->aux; u32 access_end; int err; @@ -6064,6 +6065,11 @@ static int check_buffer_access(struct bpf_verifier_env *env, return err; *max_access = max(access_end, *max_access); + if (write && size) { + if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write) + aux->min_rdwr_write = access_end - size; + aux->max_rdwr_write = max(access_end, aux->max_rdwr_write); + } return 0; } @@ -7339,7 +7345,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } err = check_buffer_access(env, reg, argno, off, size, false, - max_access); + max_access, t == BPF_WRITE); if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); @@ -7852,7 +7858,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, - max_access); + max_access, access_type == BPF_WRITE); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 72b9ee4c98de..1c59300af05c 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -925,7 +925,7 @@ static int bpf_iter_map_check_prog(const struct bpf_prog *prog, if (prog->aux->max_rdwr_access > aux->map->value_size) return -EACCES; - return 0; + return bpf_iter_map_check_value_write(prog, aux->map); } static int bpf_iter_attach_map(struct bpf_prog *prog, From 5bdd61e98da5468140aff5b5abc0e8544deaf2ee Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 16:37:32 +0000 Subject: [PATCH 585/660] bpf: Free what map_free() unlinks for sleepable readers bpf_local_storage_map_free() takes the elements of the map off the lists of their owners, which are alive, and frees them, and the storage of an owner with its last element, behind a plain RCU grace period. Nobody uses the map any more, but the other maps of these owners are in use. A sleepable program calls the get and delete helpers under rcu_read_lock_trace() only. While it walks the list of an owner for its own map, or is about to lock the storage, the element it stands on, or the storage, can be freed under it. Whatever else takes an element off the list of a live owner frees it behind an RCU Tasks Trace grace period, which implies a plain one. Do so in bpf_local_storage_map_free() as well. It waits for these callbacks before it frees the map. This does not help a sleepable program that hands an owner to the helpers which nobody holds, and which may be gone by then. Fixes: dcf456c9a095 ("bpf: Fix usage of trace RCU in local storage.") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017163745.3341787-31-alexei.starovoitov@gmail.com --- kernel/bpf/bpf_local_storage.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index c08afb33d038..49ea00c99390 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -431,13 +431,17 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * then freed after an RCU grace period. However, if destroy() and map_free() are * racing or rqspinlock returns errors in unlikely situations (unlink != 2), free * the selem only after both map_free() and destroy() see the selem. + * + * The owner that map_free() takes an selem from is alive: sleepable code, which + * is in no RCU read side critical section, may be walking its list for another + * map. */ if (unlink == 2 || atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED) - bpf_selem_free(selem, true); + bpf_selem_free(selem, !in_map_free); if (free_storage) - bpf_local_storage_free(local_storage, true); + bpf_local_storage_free(local_storage, !in_map_free); } void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, From b83bacf2161d0f8583e3f1e9c90402638992ad82 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:35 +0000 Subject: [PATCH 586/660] bpf: Don't take a program with a target program for an iterator bpf_iter_prog_supported() goes by the name of the attach function. Commit 7cc639dc45c5 ("bpf: Don't take an extension for an iterator program") keeps out extensions, for which that is the name of a function of another program. A tracing program with BPF_TRACE_ITER that is loaded with attach_prog_fd gets there as well: unlike for BPF_TRACE_RAW_TP, bpf_check_attach_target() does not mind the target program. If the function is called like that of an iterator target, bpf_iter_bpf_map for example, its id in the BTF of the program is cached as the id of the target, even if the load fails later. Programs of that target which are loaded already cannot be attached any more. And the program itself can be attached as an iterator of that target, with a context that the verifier has taken from the prototype of that function: BUG: KASAN: stack-out-of-bounds in bpf_prog_66d4234e2e50f5b6_ts26_forged+0x12a/0x20b Read of size 8 at addr ffffc90000d9fc48 by task ts26/104 Refuse every program that has a target program. That covers extensions. Fixes: 15d83c4d7cef ("bpf: Allow loading of a bpf_iter program") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-2-alexei.starovoitov@gmail.com --- kernel/bpf/bpf_iter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 5fd376bd210b..de4334bfee5d 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -352,10 +352,10 @@ int bpf_iter_prog_supported(struct bpf_prog *prog) int prefix_len = strlen(prefix); /* - * The attach function of an extension is a subprogram of the iterator - * program that it extends, whatever its name is. + * The attach function of a program that has a target program is a + * function of that program, whatever its name is. */ - if (prog->type == BPF_PROG_TYPE_EXT) + if (prog->aux->dst_prog) return -EINVAL; if (strncmp(attach_fname, prefix, prefix_len)) From fcbfa99aa657627528b75e7dc1023bcb0b84e558 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:36 +0000 Subject: [PATCH 587/660] bpf: Don't let an extension reach further into the context than its target How large the context of a raw tracepoint, tracepoint or syscall program is depends on what the program is attached to, or is run with. bpf_probe_register(), perf_event_set_bpf_prog() and the test runs compare that with max_ctx_offset of the program. An extension of the program, or of one of its global functions, gets the same context. How far it reaches is noted in its own max_ctx_offset, which nobody looks at. On a tracepoint with one argument it reads the stack behind the arguments, in a test run the memory behind the context, or from NULL if there is none: BUG: KASAN: stack-out-of-bounds in bpf_prog_954b5d2051843629_ts28_ext_far+0x7e/0x113 Read of size 8 at addr ffffc90000def7b8 by task ts28/104 BUG: KASAN: slab-out-of-bounds in bpf_prog_954b5d2051843629_ts28_ext_far+0x7e/0x113 Read of size 8 at addr ffff88810712c3e8 by task ts28/104 Refuse to attach an extension that reaches further than the program which it extends. A program that leaves the context to its extensions has to touch as much of it as they do. Fixes: be8704ff07d2 ("bpf: Introduce dynamic program extensions") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-3-alexei.starovoitov@gmail.com --- kernel/bpf/syscall.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c412bcd1903f..d026ab779ce8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3851,6 +3851,19 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -EINVAL; goto out_unlock; } + /* + * The size of these contexts depends on what the program is attached + * to, or run with. That is compared with max_ctx_offset of the program, + * and nobody looks at the one of its extensions. + */ + if (prog->type == BPF_PROG_TYPE_EXT && + (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_SYSCALL) && + prog->aux->max_ctx_offset > tgt_prog->aux->max_ctx_offset) { + err = -EINVAL; + goto out_unlock; + } err = bpf_link_prime(&link->link.link, &link_primer); if (err) From f693030a187b56a3946b36521bb90d785f906669 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:37 +0000 Subject: [PATCH 588/660] bpf: Keep values of a dead owner for sleepable programs bpf_local_storage_destroy() frees the elements of an owner behind a plain RCU grace period, on the ground that no program can refer to an owner that goes away. But the pointer that bpf_task_storage_get() and its like return stays valid for the verifier after the program has released the owner, or has left the bpf_rcu_read_lock() section it had the owner from. A sleepable program is in no RCU read side critical section, so it can write to the value after the owner has died and the element was freed: BUG: KASAN: slab-use-after-free in _copy_from_user+0x42/0xa0 Write of size 8 at addr ffff8880074404a8 by task lsuaf/55 Free the element behind an RCU Tasks Trace grace period if sleepable code may use the map. A program that a map is bound to was loaded, and counted in map->sleepable_refcnt, before it got a value from the live owner, and stays counted until a grace period behind its last run. A program can also get at a storage map by a lookup in an outer map. Then only the outer map is counted, its programs can come after the inner map was put into it, and can hold on to the inner map after it was taken out. So note in a map that it has been an inner map, and treat it like one with sleepable users from then on. The same goes for what the special fields of the value hold. The page of a uptr has to stay pinned, and a kptr which the program stores after the owner has died has to be released. Leave the fields, and the smap that it takes to free them, to bpf_selem_free_trace_rcu(), as bpf_selem_unlink() does. bpf_local_storage_map_free() waits for that callback, and leaves smap alone if it comes across such an element. Elements of such maps now wait for sleepable programs that are running when their owner dies. Other maps are freed as before. Fixes: dcf456c9a095 ("bpf: Fix usage of trace RCU in local storage.") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-4-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 1 + kernel/bpf/bpf_local_storage.c | 31 +++++++++++++++++++++++++------ kernel/bpf/map_in_map.c | 7 +++++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 22947fa192e4..a10af47978a4 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -341,6 +341,7 @@ struct bpf_map { bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; + bool was_inner_map; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 49ea00c99390..b12fcec47ffa 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -352,7 +352,8 @@ out: static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, struct bpf_local_storage_map_bucket *b) { - bool in_map_free = !!b, free_storage = false; + bool in_map_free = !!b, free_storage = false, sleepable_users = false; + bool keep_fields = false; struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; unsigned long flags; @@ -362,17 +363,26 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); if (smap) { + /* A map that is freed, or NULL here, has no program left. */ + sleepable_users = !in_map_free && + (atomic64_read(&smap->map.sleepable_refcnt) || + READ_ONCE(smap->map.was_inner_map)); b = b ? : select_bucket(smap, local_storage); err = raw_res_spin_lock_irqsave(&b->lock, flags); if (!err) { /* * Call bpf_obj_free_fields() under b->lock to make sure it is done * exactly once for an selem. Safe to free special fields immediately - * as no BPF program should be referencing the selem. + * as no BPF program should be referencing the selem, unless it + * is sleepable: see below. */ if (likely(selem_linked_to_map(selem))) { hlist_del_init_rcu(&selem->map_node); - bpf_obj_free_fields(smap->map.record, SDATA(selem)->data); + if (sleepable_users) + keep_fields = true; + else + bpf_obj_free_fields(smap->map.record, + SDATA(selem)->data); unlink++; } raw_res_spin_unlock_irqrestore(&b->lock, flags); @@ -387,7 +397,15 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * with selem1 as raw_res_spin_lock_irqsave() returns -ETIMEDOUT. */ WARN_ON_ONCE(err && in_map_free); - if (!err || in_map_free) + /* + * A sleepable program can still use, and fill, the fields of a value + * that it got before it let go of the owner. As for bpf_selem_unlink(), + * bpf_selem_free_trace_rcu() frees them, and takes smap for that: + * map_free() waits for this RCU read side critical section and then + * for the callback. It must not take smap away from an selem that + * destroy() has unlinked from the map. + */ + if (in_map_free ? err || unlink : !err && !keep_fields) RCU_INIT_POINTER(SDATA(selem)->smap, NULL); } @@ -434,11 +452,12 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * * The owner that map_free() takes an selem from is alive: sleepable code, which * is in no RCU read side critical section, may be walking its list for another - * map. + * map. And it can still use a value that it got before it let go of an owner + * that destroy() is called for. */ if (unlink == 2 || atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED) - bpf_selem_free(selem, !in_map_free); + bpf_selem_free(selem, !in_map_free && !sleepable_users); if (free_storage) bpf_local_storage_free(local_storage, !in_map_free); diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 5ee8aae7435b..0fa3f2ac0f99 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -110,10 +110,13 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; - if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) + if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) { bpf_map_inc(inner_map); - else + /* Its users are those of the outer maps from now on */ + WRITE_ONCE(inner_map->was_inner_map, true); + } else { inner_map = ERR_PTR(-EINVAL); + } return inner_map; } From b6458abbafe4bf4b992d2c5ee06ab529e8428591 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:38 +0000 Subject: [PATCH 589/660] bpf: Don't take an extension for a function that runs sleepable An extension cannot be sleepable, and it is verified as a program that runs under the RCU read lock. But it runs in the context of the function that it replaces. For a sleepable program, or a function that is called from a bpf_wq or bpf_task_work callback, that is rcu_read_lock_trace(). The verifier lets such an extension use maps and pointers that rely on RCU. Its maps are not counted as maps with sleepable users either, so that what it got from them is freed behind a plain RCU grace period: WARNING: net/core/sock_map.c:903 at __sock_hash_lookup_elem+0xe8/0xf0 BUG: KASAN: slab-use-after-free in bpf_prog_b3bad285227d855b_ext_storage+0x245/0x297 Write of size 8 at addr ffff888108717a50 by task fr/955 The verifier knows the contexts that a global function is called in. Refuse to replace one that is called without the RCU read lock, and the main function of a sleepable program. A function that sleepable code only calls under bpf_rcu_read_lock() can still be replaced. The summarization test has replaced functions of sleepable programs by ones that do not sleep. Expect that to fail. Fixes: 1e6c62a88215 ("bpf: Introduce sleepable BPF programs") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-5-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 20 ++++++++++++++++++ .../selftests/bpf/prog_tests/summarization.c | 21 +++++-------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2c812a5f1da8..d738ce6e7428 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22151,6 +22151,21 @@ static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf * return -EINVAL; } +/* + * An extension is verified as a program that runs under the RCU read lock. Does + * the function that it is to replace? + */ +static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) +{ + const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog]; + + if (!subprog) + return !prog->sleepable; + + return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] && + !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -22247,6 +22262,11 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Cannot extend a writable raw tracepoint program\n"); return -EINVAL; } + if (!func_has_rcu_read_lock(tgt_prog, subprog)) { + bpf_log(log, + "Cannot extend a function that runs without RCU read lock\n"); + return -EINVAL; + } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; diff --git a/tools/testing/selftests/bpf/prog_tests/summarization.c b/tools/testing/selftests/bpf/prog_tests/summarization.c index 6951786044ca..3e4bcf071735 100644 --- a/tools/testing/selftests/bpf/prog_tests/summarization.c +++ b/tools/testing/selftests/bpf/prog_tests/summarization.c @@ -53,17 +53,6 @@ static void test_aux(const char *main_prog_name, err = summarization_freplace__load(freplace); print_verifier_log(log); - /* The might_sleep extension doesn't work yet as sleepable calls are not - * allowed, but preserve the check in case it's supported later and then - * this particular combination can be enabled. - */ - if (!strcmp("might_sleep", replacement) && err) { - ASSERT_HAS_SUBSTR(log, "sleepable helper bpf_copy_from_user#", "error log"); - ASSERT_EQ(err, -EINVAL, "err"); - test__skip(); - goto out; - } - if (expect_load) { ASSERT_OK(err, "summarization_freplace__load"); } else { @@ -86,9 +75,8 @@ out: * Also test a case when main subprogram itself is replaced and is a single * subprogram in a program. * - * This holds for might_sleep programs. It is ok to replace might_sleep with - * might_sleep and with does_not_sleep, but does_not_sleep cannot be replaced - * with might_sleep. + * An extension is verified as a non-sleepable program, hence nothing that runs + * sleepable can be replaced. */ void test_summarization_freplace(void) { @@ -111,7 +99,7 @@ void test_summarization_freplace(void) }, }; const char *pkt_err = "Extension program changes packet data"; - const char *slp_err = "Extension program may sleep"; + const char *slp_err = "Cannot extend a function that runs without RCU read lock"; struct { const char *func; bool has_side_effect; @@ -136,7 +124,8 @@ void test_summarization_freplace(void) if (!test__start_subtest(buf)) continue; test_aux(mains[t][i].main, mains[t][i].to_be_replaced, replacements[t][j].func, - mains[t][i].has_side_effect || !replacements[t][j].has_side_effect, + t == 0 && + (mains[t][i].has_side_effect || !replacements[t][j].has_side_effect), replacements[t][j].err_msg); } } From 4eaf4d71b91a983489227336adcab33d8eae58a5 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:39 +0000 Subject: [PATCH 590/660] bpf: Keep the symbols of a program until it has ended When the last reference to a program goes, __bpf_prog_put_noref() takes its symbols out of the kallsyms tree and frees its line info at once, and waits for the grace periods only with the rest. But the program may still be running: the link or the array it was called from is gone, the call is not over. search_bpf_extables() finds the exception table of a program by its symbol. A load that faults, and that the table is there to fix up, is fatal from then on: Oops: general protection fault, probably for non-canonical address 0xdead0000000006f8: 0000 [#1] SMP NOPTI RIP: 0010:0xffffffffc06008f6 It takes a program that is detached and closed on one CPU while it runs on another. Unwinders and bpf_prog_find_from_stack() do not know the program any more either, and the stack dump of a BPF stream reads the freed line info. Free the line info behind the grace periods the program needs, as it is done with the BTF that its text comes from. Remove the symbols in bpf_prog_free_deferred(), which runs behind them as well. The symbols are looked up under rcu_read_lock(), so the images and the program itself are freed one more grace period later. All that the program refers to is released when it was before. Fixes: 3dec541b2e63 ("bpf: Add support for BTF pointers to x86 JIT") Fixes: d7c431cafcb4 ("bpf: Add dump_stack() analogue to print to BPF stderr") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-6-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 1 + kernel/bpf/core.c | 50 +++++++++++++++++++++++++++++++------------- kernel/bpf/syscall.c | 5 ++--- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a10af47978a4..feb7de6483e3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1971,6 +1971,7 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e1216927373a..6e32db9777c2 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3085,12 +3085,40 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } -static void bpf_prog_free_deferred(struct work_struct *work) +static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; int i; + aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); + for (i = 0; i < aux->real_func_cnt; i++) { + /* We can just unlink the subprog poke descriptor table as + * it was originally linked to the main program and is also + * released along with it. + */ + aux->func[i]->aux->poke_tab = NULL; + bpf_jit_free(aux->func[i]); + } + if (aux->real_func_cnt) { + kfree(aux->func); + bpf_prog_unlock_free(aux->prog); + } else { + bpf_jit_free(aux->prog); + } +} + +static void bpf_prog_free_deferred(struct work_struct *work) +{ + struct bpf_prog_aux *aux; + bool ksyms; + aux = container_of(work, struct bpf_prog_aux, work); + /* + * A prog that still runs when its last reference goes needs its symbols, + * by which its exception table is found. It has ended by now. + */ + ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); + bpf_prog_kallsyms_del_all(aux->prog); #ifdef CONFIG_BPF_SYSCALL bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); #endif @@ -3109,20 +3137,12 @@ static void bpf_prog_free_deferred(struct work_struct *work) #endif if (aux->dst_trampoline) bpf_trampoline_put(aux->dst_trampoline); - for (i = 0; i < aux->real_func_cnt; i++) { - /* We can just unlink the subprog poke descriptor table as - * it was originally linked to the main program and is also - * released along with it. - */ - aux->func[i]->aux->poke_tab = NULL; - bpf_jit_free(aux->func[i]); - } - if (aux->real_func_cnt) { - kfree(aux->func); - bpf_prog_unlock_free(aux->prog); - } else { - bpf_jit_free(aux->prog); - } + /* The symbols are looked up under rcu_read_lock() */ + INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); + if (ksyms) + queue_rcu_work(system_percpu_wq, &aux->rwork); + else + bpf_prog_free_images(&aux->rwork.work); } void bpf_prog_free(struct bpf_prog *fp) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d026ab779ce8..849bc9136299 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2461,6 +2461,8 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); btf_put(aux->btf); + kvfree(aux->jited_linfo); + kvfree(aux->linfo); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2485,10 +2487,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); module_put(prog->aux->mod); - kvfree(prog->aux->jited_linfo); - kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); if (prog->aux->attach_btf) From 6aff0129f327b159c86a54e6bea64a027630ff6b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:40 +0000 Subject: [PATCH 591/660] bpf: Use the socket of a sk_lookup test run under RCU bpf_sk_lookup_assign() accepts only sockets which are freed by RCU, and takes no reference. bpf_prog_test_run_sk_lookup() looks at sk_reuseport of the selected socket and generates its cookie, which is a write, behind bpf_test_timer_leave(), that is outside of the RCU read side critical section in which the program ran. If the socket is closed in the meantime it may be gone. Do both before the section ends. Fixes: 7c32e8f8bc33 ("bpf: Add PROG_TEST_RUN support for sk_lookup programs") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-7-alexei.starovoitov@gmail.com --- net/bpf/test_run.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index d73cd247f8cd..3043a7393c29 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1656,21 +1656,20 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat ctx.selected_sk = NULL; retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); + + /* Nothing but RCU keeps the selected socket */ + user_ctx->cookie = 0; + if (!ret && ctx.selected_sk) { + if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) + ret = -EOPNOTSUPP; + else + user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); + } bpf_test_timer_leave(&t); if (ret < 0) goto out; - user_ctx->cookie = 0; - if (ctx.selected_sk) { - if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { - ret = -EOPNOTSUPP; - goto out; - } - - user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); - } - ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration); if (!ret) ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); From f99351272933ba2e1221484a5e53b7a5d2c035f0 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:41 +0000 Subject: [PATCH 592/660] bpf, cpumap: Hold the device of frames in the queue A frame that is redirected to another CPU waits in the queue of the cpumap entry with a plain pointer to the device it came from, and an skb from generic XDP with skb->dev. Nothing keeps that device. Closing and unregistering it waits for its NAPI and for RCU, but not for the kthread of the entry. If the kthread is late by more than that, since its CPU is busy with something of a higher priority, it builds skbs for a device that has been freed and hands them to the stack: BUG: KASAN: use-after-free in dev_gro_receive+0x1f51/0x2b30 Read of size 8 at addr ffff88800a7340b0 by task cpumap/1/map:1/54 Take a reference to the device for each entry of the queue. The kthread cannot drop it at the end of the batch, since its GRO node may keep the skb for longer. Let it note the devices and drop the references behind a full GRO flush, which it does whenever the queue is empty, and from now on also when it has noted about 2048 of them. A device whose frames are stuck in a queue is now unregistered only when the kthread has got to them. RTNL is not held meanwhile, but if the device goes with its network namespace, the cleanup of all namespaces waits. Fixes: 1c601d829ab0 ("bpf: cpumap xdp_buff to skb conversion and allocation") Fixes: 11941f8a8536 ("bpf: cpumap: Implement generic cpumap") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-8-alexei.starovoitov@gmail.com --- kernel/bpf/cpumap.c | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 5e59ab896f05..eb87131b1bef 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -45,6 +45,7 @@ */ #define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */ +#define CPU_MAP_HELD_DEVS 2048 struct bpf_cpu_map_entry; struct bpf_cpu_map; @@ -74,6 +75,13 @@ struct bpf_cpu_map_entry { struct completion kthread_running; struct rcu_work free_work; + + /* + * Every frame and skb in the queue holds a reference to its device. + * The kthread keeps it until nothing is left in GRO. + */ + struct net_device *held[CPU_MAP_HELD_DEVS]; + u32 nr_held; }; struct bpf_cpu_map { @@ -129,9 +137,11 @@ static void __cpu_map_ring_cleanup(struct ptr_ring *ring) WARN_ON_ONCE(1); if (unlikely(__ptr_test_bit(0, &ptr))) { __ptr_clear_bit(0, &ptr); + dev_put(((struct sk_buff *)ptr)->dev); kfree_skb(ptr); continue; } + dev_put(((struct xdp_frame *)ptr)->dev_rx); xdp_return_frame(ptr); } } @@ -311,7 +321,7 @@ static int cpu_map_kthread_run(void *data) void *frames[CPUMAP_BATCH]; void *skbs[CPUMAP_BATCH]; u32 i, n, m; - bool empty; + bool empty, full; /* Release CPU reschedule checks */ if (__ptr_ring_empty(rcpu->queue)) { @@ -344,10 +354,12 @@ static int cpu_map_kthread_run(void *data) struct sk_buff *skb = f; __ptr_clear_bit(0, &skb); + rcpu->held[rcpu->nr_held++] = skb->dev; skbs[ret.skb_n++] = skb; continue; } + rcpu->held[rcpu->nr_held++] = ((struct xdp_frame *)f)->dev_rx; frames[ret.xdp_n++] = f; page = virt_to_page(f); @@ -398,10 +410,13 @@ stats: /* Flush either every 64 packets or in case of empty ring */ packets += n; empty = __ptr_ring_empty(rcpu->queue); - if (packets >= NAPI_POLL_WEIGHT || empty) { - cpu_map_gro_flush(rcpu, empty); + full = empty || rcpu->nr_held > CPU_MAP_HELD_DEVS - CPUMAP_BATCH; + if (packets >= NAPI_POLL_WEIGHT || full) { + cpu_map_gro_flush(rcpu, full); packets = 0; } + while (full && rcpu->nr_held) + dev_put(rcpu->held[--rcpu->nr_held]); local_bh_enable(); /* resched point, may call do_softirq() */ } @@ -738,12 +753,14 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) for (i = 0; i < bq->count; i++) { struct xdp_frame *xdpf = bq->q[i]; - int err; - err = __ptr_ring_produce(q, xdpf); - if (err) { + if (__ptr_ring_full(q)) { drops++; xdp_return_frame_rx_napi(xdpf); + } else { + /* The device cannot go before the flush, from now on it can */ + dev_hold(xdpf->dev_rx); + __ptr_ring_produce(q, xdpf); } processed++; } @@ -807,11 +824,15 @@ int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, __skb_pull(skb, skb->mac_len); skb_set_redirected(skb, false); + dev_hold(skb->dev); __ptr_set_bit(0, &skb); ret = ptr_ring_produce(rcpu->queue, skb); - if (ret < 0) + if (ret < 0) { + __ptr_clear_bit(0, &skb); + dev_put(skb->dev); goto trace; + } wake_up_process(rcpu->kthread); trace: From 19e6ca6b4a22d62a6f1dc54f230714adb4396279 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:42 +0000 Subject: [PATCH 593/660] bpf: Drop packet pointers behind kfuncs writing an skb bpf_dynptr_write() to the dynptr of an skb ends in __bpf_skb_store_bytes(), which makes the range writable first. That pulls data into the head or unclones it, and either can reallocate the head. So check_helper_call() invalidates all packet pointers and all slices of the skb behind the helper. Several kfuncs take the same road: bpf_dynptr_copy() and bpf_dynptr_memset(), if they get no writable slice of the destination because it is not linear, and bpf_probe_read_{user,kernel}[_str]_dynptr() and bpf_copy_from_user[_task][_str]_dynptr() likewise. The verifier does not know. A program can keep a packet pointer or a slice across such a call and read and write the old head, which has been freed. Treat a kfunc of that kind as bpf_dynptr_write() is treated, by the type of the dynptr it writes to, which is the first argument of all of them. Where a global function is only scanned for what it may do, the type is not known yet. There any call of these kfuncs counts, as a tail call does. Fixes: daec295a7094 ("bpf/helpers: Introduce bpf_dynptr_copy kfunc") Fixes: a498ee7576de ("bpf: Implement dynptr copy kfuncs") Fixes: 5fc5d8fded57 ("bpf: Add bpf_dynptr_memset() kfunc") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-9-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/verifier.c | 44 ++++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e420d211f423..7a72bb06fe56 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1803,6 +1803,8 @@ struct bpf_call_arg_meta { u32 kfunc_flags; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d738ce6e7428..597302cf7086 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1002,6 +1002,11 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; } +static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) +{ + return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; +} + static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { @@ -8976,6 +8981,7 @@ static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta); static bool is_bpf_obj_drop_kfunc(u32 func_id); static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); static bool is_bpf_rbtree_add_kfunc(u32 func_id); @@ -10040,6 +10046,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; + if (!arg && is_kfunc_dynptr_write(meta)) + meta->dynptr_write_type = meta->dynptr.type; break; } case ARG_PTR_TO_ITER: @@ -12654,8 +12662,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT; - if (dynptr_type == BPF_DYNPTR_TYPE_SKB || - dynptr_type == BPF_DYNPTR_TYPE_SKB_META) + if (dynptr_write_changes_pkt_data(dynptr_type)) /* this will trigger clear_all_pkt_pointers(), which will * invalidate all dynptr slices associated with the skb */ @@ -13704,9 +13711,36 @@ static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); } +/* These write to what their first argument, a dynptr, points to */ +BTF_SET_START(dynptr_write_kfuncs) +BTF_ID(func, bpf_dynptr_copy) +BTF_ID(func, bpf_dynptr_memset) +#ifdef CONFIG_BPF_EVENTS +BTF_ID(func, bpf_probe_read_user_dynptr) +BTF_ID(func, bpf_probe_read_kernel_dynptr) +BTF_ID(func, bpf_probe_read_user_str_dynptr) +BTF_ID(func, bpf_probe_read_kernel_str_dynptr) +BTF_ID(func, bpf_copy_from_user_dynptr) +BTF_ID(func, bpf_copy_from_user_str_dynptr) +BTF_ID(func, bpf_copy_from_user_task_dynptr) +BTF_ID(func, bpf_copy_from_user_task_str_dynptr) +#endif +BTF_SET_END(dynptr_write_kfuncs) + +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta) +{ + return meta->btf == btf_vmlinux && + btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id); +} + +/* + * A write through the dynptr of an skb may reallocate its head. Before the + * arguments are checked nobody knows what kind of dynptr it is. + */ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { - return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]); + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) || + is_kfunc_dynptr_write(meta); } static u32 kfunc_abi_slots(const struct btf_func_model *fm) @@ -15644,7 +15678,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (bpf_is_kfunc_pkt_changing(&meta)) + if (is_kfunc_dynptr_write(&meta) ? + dynptr_write_changes_pkt_data(meta.dynptr_write_type) : + bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); proto_slots = kfunc_abi_slots(&desc->func_model); From 589998616e0f9190ba6f6644a19532c5b2ba66d0 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:43 +0000 Subject: [PATCH 594/660] bpf: Let bpf_dynptr_write() mark a global function The caller of a global function loses its packet pointers if the function may change packet data. That is found out by a scan for calls of the helpers which bpf_helper_changes_pkt_data() lists. bpf_dynptr_write() is not on the list, because it depends on the type of the dynptr: check_helper_call() looks at it. The scan cannot do that, and so a global function which makes a dynptr from the skb, or gets one as an argument, and writes through it counts as harmless. The write can reallocate the head of the skb, and the caller goes on with pointers into the old one, which has been freed. Let every call of bpf_dynptr_write() mark the function, as a tail call does. The same mark decides whether a program may replace a function. Fixes: 51081a3f25c7 ("bpf: track changes_pkt_data property for global functions") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-10-alexei.starovoitov@gmail.com --- kernel/bpf/cfg.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index bce7b5abcfda..2ee950e9ac82 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -598,7 +598,9 @@ static int visit_insn(int t, struct bpf_verifier_env *env) */ if (ret == 0 && fp->might_sleep) mark_subprog_might_sleep(env, t); - if (bpf_helper_changes_pkt_data(insn->imm)) + /* nobody knows yet whether the dynptr is that of an skb */ + if (bpf_helper_changes_pkt_data(insn->imm) || + insn->imm == BPF_FUNC_dynptr_write) mark_subprog_changes_pkt_data(env, t); } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_call_arg_meta meta; From b7f97db88948aa511bcd989978f181e1367e0e8b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:44 +0000 Subject: [PATCH 595/660] bpf: Don't take a dynptr argument for a local dynptr A global function is verified on its own. A dynptr which it gets as an argument is assumed to be of type BPF_DYNPTR_TYPE_LOCAL, but the callers may pass one of any type, that of an skb for example. The type decides whether a write through the dynptr invalidates packet pointers, and whether a slice of it is invalidated together with them. Inside of the function neither happens. It can take a packet pointer from its context argument, or a slice from the dynptr, write through the dynptr and go on using the pointer, although the write may have reallocated the head of the skb. Assume the type of an skb dynptr instead. Nothing becomes forbidden for the dynptr itself, and a clone inherits the caution. The dynptr which the callback of bpf_user_ringbuf_drain() gets is known to be local, and stays so. Fixes: a64bfe618665 ("bpf: add support for passing dynptr pointer to global subprog") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-11-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 597302cf7086..338376492647 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21589,8 +21589,12 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - /* assume unspecial LOCAL dynptr type */ - __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); + /* + * The callers may hand in a dynptr of any type. + * Take it for that of an skb, so that a write + * through it and its slices are treated with care. + */ + __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & From aa486cff4964d0cd566954cec2b01b484641f7d5 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:45 +0000 Subject: [PATCH 596/660] bpf: Refuse the prologue for an extension that cannot take it A tc, lwt_xmit or sk_skb program that writes to packet data starts with bpf_unclone_prologue(). It takes the skb from R1, and if the skb is cloned it calls bpf_skb_pull_data() and restores R1 only. That suits the main function of a program. An extension gets the prologue too, but it starts with the arguments of the function that it replaces. If the first one is not the context, something else is taken for an skb: BUG: KASAN: stack-out-of-bounds in skb_ensure_writable+0x84/0x1a0 Read of size 8 at addr ffffc90000270ad0 by task uncl/105 skb_ensure_writable+0x84/0x1a0 bpf_skb_pull_data+0x32/0xf0 bpf_prog_853eb1991d35e093_ext_slice+0x4f/0x321 bpf_prog_9eb18752b5d0959a_t_glob+0x2ac/0x2de BUG: unable to handle page fault for address: 0000000082298821 And whatever R2-R5 held is lost, while the verifier goes on trusting pointers in them. Refuse an extension that needs a prologue unless it gets the context as its first argument and scalars beside it. A scalar is still clobbered if the skb is cloned, which the verifier can cope with. Extensions of XDP, sk_msg and sockopt programs get no prologue and load as before. Fixes: be8704ff07d2 ("bpf: Introduce dynamic program extensions") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-12-alexei.starovoitov@gmail.com --- kernel/bpf/fixups.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index fe59e3114d82..a8739fbd0349 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -858,6 +858,25 @@ apply_patch_buffer: return 0; } +/* + * The prologue takes the context from R1, and it can clobber R2-R5. An + * extension gets the arguments of the function that it replaces. + */ +static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) +{ + const struct bpf_subprog_info *sub = &env->subprog_info[0]; + int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + + if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) + return false; + + for (i = 1; i < cnt; i++) + if (sub->args[i].arg_type != ARG_SCALAR) + return false; + + return true; +} + /* convert load instructions that access fields of a context type into a * sequence of instructions that access fields of the underlying structure: * struct __sk_buff -> struct sk_buff @@ -915,6 +934,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + !ext_prog_takes_prologue(env)) { + verbose(env, + "Extension writing packet data must take ctx and scalars only\n"); + return -EINVAL; + } new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, BPF_PATCH_MOVE_TARGET); if (!new_prog) From bb72c2866831b571249929714d118ccea1ecb747 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:46 +0000 Subject: [PATCH 597/660] bpf: Unclone the skb for kfuncs that write through a dynptr bpf_dynptr_copy(), bpf_dynptr_memset() and the kfuncs that read user or kernel memory into a dynptr ask bpf_dynptr_slice_rdwr() for a slice and write to it. For the linear part of an skb that is a pointer into its head. bpf_dynptr_slice_rdwr() counts on the verifier to have uncloned the skb, see bpf_unclone_prologue(). The verifier does so if the program itself calls bpf_dynptr_slice_rdwr() or writes to the packet. A program that only uses these kfuncs gets no prologue, and changes the data of all the clones of the skb. A packet socket, whose clone is taken before tc ingress runs, sees what bpf_dynptr_memset() has written there. Treat a call that may write to an skb like a direct write. Fixes: daec295a7094 ("bpf/helpers: Introduce bpf_dynptr_copy kfunc") Fixes: a498ee7576de ("bpf: Implement dynptr copy kfuncs") Fixes: 5fc5d8fded57 ("bpf: Add bpf_dynptr_memset() kfunc") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-13-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 338376492647..3bc1df7aabe2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15683,6 +15683,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); + /* + * These kfuncs write to the head of the skb in place if what they write + * to is linear. Like a direct write that takes bpf_unclone_prologue(). + */ + if (is_kfunc_dynptr_write(&meta) && + dynptr_write_changes_pkt_data(meta.dynptr_write_type)) + may_access_direct_pkt_data(env, NULL, BPF_WRITE); + proto_slots = kfunc_abi_slots(&desc->func_model); if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); From 1c19f2b3b2f3b55c8017092257634cb8229b7d97 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:47 +0000 Subject: [PATCH 598/660] bpf: Follow the head if an skb is written from a slice The source of bpf_dynptr_write() may be a slice of the skb which is written to, that is a pointer into its head. bpf_dynptr_copy() does it by itself for a copy inside of one skb, if the source is linear and the destination is not. __bpf_skb_store_bytes() and __bpf_skb_meta_store_bytes() make the skb writable first. If that reallocates the head, the source points to the old one, and freed memory is copied into the packet. The offsets in the head do not change when it is reallocated here. Move the source along with the head. Fixes: 2012c867c800 ("bpf: verifier: Accept dynptr mem as mem in helpers") Fixes: f38499ff45f5 ("bpf: Unclone skb head on bpf_dynptr_write to skb metadata") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-14-alexei.starovoitov@gmail.com --- net/core/filter.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index caa9c3ef0a78..0abedbb04ab7 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1715,6 +1715,24 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); } +/* + * The source of a write through a dynptr may be a slice of the same skb. + * Making the skb writable can move its head: follow the move. + */ +static int bpf_try_make_writable_from(struct sk_buff *skb, u32 write_len, + const void **from) +{ + const void *head = skb->head; + bool in_head = *from >= head && + *from < (const void *)skb_end_pointer(skb); + int err; + + err = bpf_try_make_writable(skb, write_len); + if (!err && in_head) + *from = skb->head + (*from - head); + return err; +} + BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, const void *, from, u32, len, u64, flags) { @@ -1724,7 +1742,7 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, return -EINVAL; if (unlikely(offset > INT_MAX)) return -EFAULT; - if (unlikely(bpf_try_make_writable(skb, offset + len))) + if (unlikely(bpf_try_make_writable_from(skb, offset + len, &from))) return -EFAULT; ptr = skb->data + offset; @@ -12427,7 +12445,7 @@ int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset, { if (unlikely(flags)) return -EINVAL; - if (unlikely(bpf_try_make_writable(skb, 0))) + if (unlikely(bpf_try_make_writable_from(skb, 0, &from))) return -EFAULT; memmove(bpf_skb_meta_pointer(skb, offset), from, len); From 1db084f42884c5af881fedf3df4777fe2095b20b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:48 +0000 Subject: [PATCH 599/660] bpf: Use memmove() to store bytes in an skb Through bpf_dynptr_write() and bpf_dynptr_copy() the source of bpf_skb_store_bytes() can be a slice of the same skb which overlaps the destination. memcpy() then reads bytes which it has overwritten already. On x86 with FSRM it repeats the first bytes of the source instead of copying it. Local, ringbuf and skb metadata dynptrs are written with memmove() for that reason. Fixes: 2012c867c800 ("bpf: verifier: Accept dynptr mem as mem in helpers") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-15-alexei.starovoitov@gmail.com --- net/core/filter.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/core/filter.c b/net/core/filter.c index 0abedbb04ab7..7fbee7570d0c 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1749,7 +1749,8 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpull_rcsum(skb, ptr, len, offset); - memcpy(ptr, from, len); + /* through a dynptr the source may be an overlapping slice of the skb */ + memmove(ptr, from, len); if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpush_rcsum(skb, ptr, len, offset); From ab08863c58b1a96ff8378e373cb3d277cbe65b68 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 17 Oct 2026 23:09:49 +0000 Subject: [PATCH 600/660] bpf: Use memmove() to store bytes in an xdp_buff bpf_dynptr_write() to the dynptr of an xdp_buff ends in bpf_xdp_store_bytes(). The source can be a slice of the same xdp_buff which overlaps the destination, and the helper itself takes such a slice as well. memcpy() then reads bytes which it has overwritten already. On x86 with FSRM p = bpf_dynptr_slice_rdwr(&ptr, 0, NULL, 64); bpf_dynptr_write(&ptr, 8, p, 64, 0); repeats the first 8 bytes of the packet instead of moving 64. A write that crosses from the linear part into a fragment is copied piece by piece, where the first piece can overwrite what the second one has yet to read. That is not dealt with here. Fixes: 2012c867c800 ("bpf: verifier: Accept dynptr mem as mem in helpers") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-16-alexei.starovoitov@gmail.com --- net/core/filter.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/core/filter.c b/net/core/filter.c index 7fbee7570d0c..b7ac145b68b1 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4330,7 +4330,8 @@ BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset, if (!ptr) bpf_xdp_copy_buf(xdp, offset, buf, len, true); else - memcpy(ptr, buf, len); + /* through a dynptr the source may be an overlapping slice */ + memmove(ptr, buf, len); return 0; } From a6f8a67c13ddd5fcfee0fae61ec5787ee6729ab3 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Sat, 17 Oct 2026 23:09:50 +0000 Subject: [PATCH 601/660] bpf: Refuse packet slices for packet-changing helpers The helpers which may change packet data do not take a packet pointer as a memory argument: they have no pkt_access. They make the skb writable, or make room in it, before they read what the argument points to, and that can reallocate the head which it points into. A slice from bpf_dynptr_slice() of the dynptr of the skb points into the same head, but it is a PTR_TO_MEM, and the verifier lets it pass. bpf_lwt_push_encap(), bpf_lwt_seg6_store_bytes() and bpf_lwt_seg6_action() then read the old head, which has been freed. bpf_skb_store_bytes() follows the head by now, but a slice is no better than a packet pointer there. Refuse a slice of a packet where a packet pointer is refused for that reason. All other helpers take slices as before. bpf_dynptr_write() and bpf_dynptr_copy() can be used to copy inside of a packet. Fixes: 2012c867c800 ("bpf: verifier: Accept dynptr mem as mem in helpers") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261017230950.802052-17-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3bc1df7aabe2..63fff641d985 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9687,6 +9687,16 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p if (err) return err; + /* + * A slice of a packet is packet data under another name. A helper which + * may reallocate the packet must not go on reading from the old one. + */ + if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) && + bpf_helper_changes_pkt_data(meta->func_id)) { + verbose(env, "function access to the packet is not allowed\n"); + return -EACCES; + } + if (arg_type_is_raw_mem(arg_type)) meta->arg_raw_mem.mask |= BIT(slot); From 73a5accbfd210bda7e88e969310f15f8e515dc86 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Sun, 18 Oct 2026 13:33:52 +0200 Subject: [PATCH 602/660] bpf, x86: Mask the shift counts of the magic number division The kernel test robot reports for a clang 22 randconfig with UBSAN and ThinLTO: vmlinux.o: error: objtool: do_jit(): unexpected end of section .text.do_jit Fix by masking the counts to their range like rol64() does, which keeps the shifts defined for the sanitizer and removes the __ubsan_handle_shift_out_of_bounds() checks. The counts are below 32 and 64 anyway, so the emitted code does not change. Reproduced with clang 20 and the robot's config without LTO. Fixes: 02cfd66f3d9a ("bpf, x86: Divide by constants via multiplication with a magic number") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610171029.DfoBvNXv-lkp@intel.com/ Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261018113352.730799-1-daniel@iogearbox.net --- arch/x86/net/bpf_jit_comp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 11a4e4f87aac..d3843045dc71 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2080,7 +2080,7 @@ struct bpf_div_magic { */ static void bpf_magic_udiv(u64 d, int lz, struct bpf_div_magic *mag) { - u64 mask = ~0ULL >> lz; + u64 mask = ~0ULL >> (lz & 63); u64 nc, delta, q1, r1, q2, r2; int p; @@ -2200,7 +2200,7 @@ static bool emit_div_const(u8 **pprog, const struct bpf_insn *insn, u32 dst_reg) * the add sequence. */ if (mag.add && !(imm32 & 1)) { - pre = __ffs(imm32); + pre = __ffs(imm32) & 31; bpf_magic_udiv(imm32 >> pre, pre, &mag); if (mag.add) return false; From 8cebaa3be829c88a59957900f8de98fed09daefe Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Sun, 18 Oct 2026 09:29:07 -0400 Subject: [PATCH 603/660] locking/atomic: arc, hexagon, openrisc: Define arch_cmpxchg_local() Commit ccb88db8dac8 ("bpf: Don't hand out a linked element as extra element") made alloc_htab_elem() take the per-cpu spare element with try_cmpxchg_local(), the first caller of cmpxchg_local() outside of architecture code. It doesn't build on arc, hexagon and openrisc: include/linux/atomic/atomic-arch-fallback.h:384:27: error: implicit declaration of function 'arch_cmpxchg_local'; did you mean 'raw_cmpxchg_local'? [-Werror=implicit-function-declaration] raw_cmpxchg_local() maps to arch_cmpxchg_local() unconditionally, but these three architectures don't define it, and unlike nios2 and microblaze they don't get it from asm-generic/cmpxchg.h either. cmpxchg_local() only has to be atomic with respect to the local CPU, so an architecture's own arch_cmpxchg() can serve for the sizes it supports. arc uses its relaxed LLOCK/SCOND variant when ARC_HAS_LLSC is set and its atomic_ops_lock() variant otherwise, and openrisc uses its LL/SC arch_cmpxchg(), the way arm64, riscv and s390 alias theirs. The hexagon arch_cmpxchg() only handles 32-bit words, so hexagon uses the generic version that disables interrupts, like sh, sparc32 and xtensa do. Linux has no NMIs on hexagon, so that is sufficient there. Fixes: ccb88db8dac8 ("bpf: Don't hand out a linked element as extra element") Reported-by: kernelci.org bot Reported-by: kernel test robot Signed-off-by: Sasha Levin Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018132907.3823160-1-sashal@kernel.org Closes: https://d.kernelci.org/i/maestro:78b65519ee4561df1fbf1e72f1dec9da7e5b153e Closes: https://lore.kernel.org/oe-kbuild-all/202610181750.BoqAddaF-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202610182048.aEaiBP8W-lkp@intel.com/ --- arch/arc/include/asm/cmpxchg.h | 4 ++++ arch/hexagon/include/asm/cmpxchg.h | 6 ++++++ arch/openrisc/include/asm/cmpxchg.h | 2 ++ 3 files changed, 12 insertions(+) diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h index 66b3743c9400..93f7153eb267 100644 --- a/arch/arc/include/asm/cmpxchg.h +++ b/arch/arc/include/asm/cmpxchg.h @@ -59,6 +59,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg_relaxed + #else #define arch_cmpxchg(ptr, old, new) \ @@ -80,6 +82,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg + #endif /* diff --git a/arch/hexagon/include/asm/cmpxchg.h b/arch/hexagon/include/asm/cmpxchg.h index 9c58fb81f7fd..e6114e713db9 100644 --- a/arch/hexagon/include/asm/cmpxchg.h +++ b/arch/hexagon/include/asm/cmpxchg.h @@ -8,6 +8,8 @@ #ifndef _ASM_CMPXCHG_H #define _ASM_CMPXCHG_H +#include + /* * __arch_xchg - atomically exchange a register and a memory location * @x: value to swap @@ -72,4 +74,8 @@ __arch_xchg(unsigned long x, volatile void *ptr, int size) __oldval; \ }) +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), \ + (unsigned long)(o), (unsigned long)(n), sizeof(*(ptr)))) + #endif /* _ASM_CMPXCHG_H */ diff --git a/arch/openrisc/include/asm/cmpxchg.h b/arch/openrisc/include/asm/cmpxchg.h index 8ee151c072e4..fcf1bc8db8fe 100644 --- a/arch/openrisc/include/asm/cmpxchg.h +++ b/arch/openrisc/include/asm/cmpxchg.h @@ -140,6 +140,8 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, sizeof(*(ptr))); \ }) +#define arch_cmpxchg_local arch_cmpxchg + /* * This function doesn't exist, so you'll get a linker error if * something tries to do an invalidly-sized xchg(). From 1c1dda9ba9f3af9cd511a197dcc86de2465d8fe5 Mon Sep 17 00:00:00 2001 From: Woojin Ji Date: Sun, 18 Oct 2026 13:21:41 +0900 Subject: [PATCH 604/660] bpf: Zero-extend the index in inlined bpf_get_func_arg() bpf_get_func_arg() takes a u32 index, but the inlined expansion compared only the low 32 bits of R2 with the argument count and then shifted the full 64-bit register to compute the context address. R2 is ARG_ANYTHING, so nothing guarantees that its upper bits are zero: an index of 1ULL << 32 passes the bounds check as index zero while the load is performed at ctx + (1ULL << 35). Shift the index as a 32-bit value. When the 32-bit bounds check passes, the low 32 bits of R2 are below R0, and R0 is at most 255: the trampoline arm masks the argument count with 0xFF and the raw_tp arm uses the BTF argument count minus one, which the verifier caps at MAX_BPF_FUNC_ARGS. The 3-bit shift therefore cannot overflow and it clears the upper bits of the register. Widening the comparison instead would reject indices whose low 32 bits are valid. The check itself is unchanged, so this cannot change which programs are accepted, only the address that is computed. Fixes: f92c1e183604 ("bpf: Add get_func_[arg|ret|arg_cnt] helpers") Signed-off-by: Woojin Ji Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018-get-func-arg-v2-1-048848b3eddf@gmail.com --- kernel/bpf/fixups.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index a8739fbd0349..713f14d73b06 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2489,7 +2489,7 @@ patch_map_ops_generic: cnt = 2; } insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); - insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); From 38dbc3047a2fa706f6b4485679eb5eb62ff55b87 Mon Sep 17 00:00:00 2001 From: Woojin Ji Date: Sun, 18 Oct 2026 13:21:42 +0900 Subject: [PATCH 605/660] selftests/bpf: Test out-of-range index bits in bpf_get_func_arg() The inlined expansion of bpf_get_func_arg() bounds-checks only the low 32 bits of the index, so a helper prototype that keeps the upper bits of the index is needed to reach it. Add check_wide_index() and call it from the fexit and tp_btf tests. It drives the helper with indices that have upper bits set and with indices whose low 32 bits are out of range, and it checks that the destination is left untouched when the helper reports -EINVAL. Without the previous patch the wide index is shifted as 64-bit and the load uses an address far outside the context. Signed-off-by: Woojin Ji Signed-off-by: Alexei Starovoitov Acked-by: Jiri Olsa Link: https://patch.msgid.link/20261018-get-func-arg-v2-2-048848b3eddf@gmail.com --- .../selftests/bpf/progs/get_func_args_test.c | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/get_func_args_test.c b/tools/testing/selftests/bpf/progs/get_func_args_test.c index 1bf47f64d096..9b0beb58d88d 100644 --- a/tools/testing/selftests/bpf/progs/get_func_args_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_args_test.c @@ -6,6 +6,40 @@ char _license[] SEC("license") = "GPL"; +/* Keep the upper index bits: the inlined expansion must truncate them. */ +static long (*bpf_get_func_arg_wide)(void *ctx, __u64 n, __u64 *value) = + (void *)BPF_FUNC_get_func_arg; + +/* Both traced functions are called with arguments 2 and 3. */ +static __always_inline int check_wide_index(void *ctx) +{ + __u64 a = 0, z = 0xdeadbeefdeadbeefULL; + __s64 err; + int ok = 1; + + /* The index is truncated to u32, so these select arguments 0 and 1. */ + err = bpf_get_func_arg_wide(ctx, 1ULL << 32, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 1, &a); + ok &= err == 0 && a == 3; + err = bpf_get_func_arg_wide(ctx, 1ULL << 59, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 1, &a); + ok &= err == 0 && a == 3; + + /* Out of range: -EINVAL and no store into the destination. */ + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + + return ok; +} + __u64 test1_result = 0; SEC("fentry/bpf_fentry_test1") int BPF_PROG(test1) @@ -63,6 +97,7 @@ int BPF_PROG(test2) /* return value */ err = bpf_get_func_ret(ctx, &ret); test2_result &= err == 0 && ret == 5; + test2_result &= check_wide_index(ctx); return 0; } @@ -162,6 +197,7 @@ int BPF_PROG(tp_test2) /* not valid argument */ err = bpf_get_func_arg(ctx, 2, &z); test6_result &= err == -EINVAL; + test6_result &= check_wide_index(ctx); return 0; } From 0007654a7bf13ef2a19afd398bbc6507087f19fd Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Sat, 17 Oct 2026 20:07:57 -0400 Subject: [PATCH 606/660] lib/buildid: reject file offsets that do not fit in pgoff_t On a 32-bit kernel, parsing the build ID of a crafted ELF file can copy far more than the intended bytes into a 64-byte on-stack buffer in __build_id_parse(), corrupting the stack of whatever context fetches the build ID. This might be reachable without privilege: perf records a build ID for PERF_RECORD_MMAP2 on the no-fault path when a process mmaps its own file (build_id_parse_nofault() from perf_event_mmap()), and BPF build-id stack maps do the same. freader_get_folio() turns a 64-bit file offset into a page cache index with file_off >> PAGE_SHIFT and passes it to filemap_get_folio(). pgoff_t is unsigned long, 32-bit on a 32-bit kernel, so an ELF64 e_phoff of 2^44 + k * PAGE_SIZE truncates to index k. filemap_get_folio() returns the cached folio at index k, whose folio_pos() is far below the requested file_off. In freader_fetch() the folio does not contain file_off, so the "crossing folio boundaries" path computes part_sz = folio_off + folio_sz - file_off as a large u64 (file_off is much larger), and the following memcpy() truncates it to size_t and copies a page into the 64-byte buffer. On a 64-bit kernel the index is not truncated, the folio is not cached and the fetch returns -EFAULT, so only 32-bit kernels are affected. Reject a file offset that cannot be represented as a page cache index before the lookup, so such a file fails the build ID parse cleanly as it already does on 64-bit. Reproduced on i386 under QEMU with CONFIG_STACKPROTECTOR_STRONG: mmapping a crafted ELF (ELFCLASS64, e_phoff = 2^44) while a perf event with attr.build_id is open panics with "stack-protector: Kernel stack is corrupted in: __build_id_parse"; with this patch the parse returns -EFAULT and the mmap completes cleanly. Built for i386 with W=1, no new warnings. Fixes: de3ec364c3c3 ("lib/buildid: add single folio-based file reader abstraction") Assisted-by: LLM Cc: stable@vger.kernel.org Signed-off-by: Nickolai Zeldovich Signed-off-by: Andrii Nakryiko --- lib/buildid.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/buildid.c b/lib/buildid.c index c4b737640621..995a327f46ae 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -47,6 +47,10 @@ static int freader_get_folio(struct freader *r, loff_t file_off) freader_put_folio(r); + /* the page index must fit in pgoff_t */ + if (file_off < 0 || file_off > MAX_LFS_FILESIZE) + return -EFAULT; + /* only use page cache lookup - fail if not already cached */ r->folio = filemap_get_folio(r->file->f_mapping, file_off >> PAGE_SHIFT); From 0ceb2f21d3e02b310ed246a210b6cecea24e19b2 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Sun, 18 Oct 2026 08:53:45 +0800 Subject: [PATCH 607/660] bpf: Determine freplace staticness from BTF linkage bpf_check_attach_target() rejects a freplace whose target is a static function, using the subprog "unreliable" flag as a proxy for "static". That flag is raised whenever a subprogram's BTF prototype disagrees with the registers at its call site. `147400e9c27c ("bpf: Compute parameters liveness in bpf_compute_live_registers()")` made that work for static callees: a static function with an unused argument no longer matches its BTF prototype at the call site and is marked unreliable. Since LLVM commit `29b36a16f111 ("[BPF] Emit ABI-accurate BTF prototypes for DW_CC_nocall")` in 23.1 [1] that premise is gone: BTFDebug may now drop arguments removed by the optimizer, so for __noinline static int subprog(struct __sk_buff *ctx) { return bpf_ktime_get_ns(); } compiled with clang 23 the BTF describes subprog() as [75] FUNC_PROTO '(anon)' ret_type_id=2 vlen=0 [76] FUNC 'subprog' type_id=75 linkage=static There is then no argument left to compare, the subprog stays reliable, and the check in bpf_check_attach_target() is skipped. The freplace is then rejected by the later linkage check instead, reporting "subprog() is not a global function", and the selftest "exceptions" fails. Determine staticness from the BTF linkage instead, and keep the unreliable check for global functions whose BTF does not match their call site. There is no functional change: freplacing a static function was rejected before and is still rejected with the same -EINVAL. Selftest "exceptions" is updated to match. [1] https://github.com/llvm/llvm-project/pull/198426 Fixes: 147400e9c27c ("bpf: Compute parameters liveness in bpf_compute_live_registers()") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018005348.2195130-2-kafai.wan@linux.dev --- kernel/bpf/verifier.c | 3 ++- tools/testing/selftests/bpf/prog_tests/exceptions.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 63fff641d985..74e5ac248a10 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22301,7 +22301,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { - if (conservative) { + if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL || + conservative) { bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index cea01d638f6f..1bd15c4c125b 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -333,7 +333,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "throwing_subprog"), "set_attach_target")) goto done; - }), "throwing_subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> non-throwing subprog")) /* non-throwing extension -> non-throwing global subprog : OK */ From 8d637d4c62c1880062dbb098f67b804e2eff85aa Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Sun, 18 Oct 2026 08:53:46 +0800 Subject: [PATCH 608/660] selftests/bpf: Fix stack_arg precision backtracking log expectation Commit 147400e9c27c ("bpf: Compute parameters liveness in bpf_compute_live_registers()") stopped assuming that a call to a static subprogram reads r1..r5 and now uses the callee's entry live-in. R1-R5 are not live at these call insns, so __clean_func_state() clears them from the caller state and the backtracked parent state prints as: mark_precise: frame0: parent state regs= stack=: R10=fp0 Update both expectations accordingly. Fixes: 147400e9c27c ("bpf: Compute parameters liveness in bpf_compute_live_registers()") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018005348.2195130-3-kafai.wan@linux.dev --- tools/testing/selftests/bpf/progs/stack_arg_precision.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index 7baa9c1c8e31..c9564776d733 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -74,7 +74,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx 11") __msg("mark_precise: frame0: regs= stack= before 9: (05) goto pc+1") __msg("mark_precise: frame0: regs= stack= before 8: (7a) *(u64 *)(r11 -8) = 4") @@ -95,7 +95,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13 ") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx 11 ") __msg("mark_precise: frame0: regs= stack= before 10: (7a) *(u64 *)(r11 -8) = 6") __naked void stack_arg_precision_bpf2bpf(void) From 8342d47ad791d70a42ab9cf262bd477fb41fdff2 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Sun, 18 Oct 2026 08:53:47 +0800 Subject: [PATCH 609/660] selftests/bpf: Fix stack arg pruning test JOIN log expectation Commit f414d20879ae ("bpf: Save DFS postorder numbers for program instructions") changed bpf_compute_postorder() to use a real DFS, which changes the postorder of this test's instructions and thus the insn edge the arg track join at insn 10 is reported on: arg JOIN insn 8 -> 10 r1: _ + fp0-8 => fp0-8|fp0+0 arg JOIN insn 8 -> 10 sa0: _ + fp0-8 => fp0-8|fp0+0 Update the r1 and sa0 expectations to the new output. Fixes: f414d20879ae ("bpf: Save DFS postorder numbers for program instructions") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018005348.2195130-4-kafai.wan@linux.dev --- tools/testing/selftests/bpf/progs/verifier_stack_arg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index f6a4a6aa0e61..b9f799f6e405 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -117,8 +117,8 @@ SEC("tc") __description("stack_arg: pruning with different stack arg types") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("arg JOIN insn 9 -> 10 r1: fp0-8 + _ => fp0-8|fp0+0") -__msg("arg JOIN insn 9 -> 10 sa0: fp0-8 + _ => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 r1: _ + fp0-8 => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 sa0: _ + fp0-8 => fp0-8|fp0+0") __msg("R{{[0-9]}} invalid mem access 'scalar'") __naked void stack_arg_pruning_type_mismatch(void) { From ce42271dc517baa880352d76548d301850e7385b Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Sun, 18 Oct 2026 08:53:48 +0800 Subject: [PATCH 610/660] selftests/bpf: Use _exit() in uprobe_multi_test clone-ed child child_func() terminates with exit(), but spawn_child_flag() also starts it via clone(CLONE_VM), which shares the parent's address space. That exit() then runs atexit handlers and static destructors against globals the parent is still using. LLVM commit `3c3ed59c6a2a ("[llvm] Add format check for MCSubtargetFeatures")` [1], included in the LLVM 23.1 releases, added Target::isValidFeatureListFormat() with a function-local static holding the compiled regex: static const llvm::Regex pattern("^([+-][^,]+)(,[+-][^,]+)*,?$"); return pattern.match(Features); The child exit destroys that static while its guard variable still marks it as initialized, so the regex is never rebuilt in the parent. From that point on, get_jited_program_text() gets NULL from LLVMCreateDisasmCPUFeatures(), so every jited program text check in the tests running after filter_clone_vm fails, e.g. "verifier_jit_convergence", "verifier_private_stack", "verifier_tailcall_jit" and "verifier_jit_inline". Use _exit(), which runs neither atexit handlers nor static destructors, as uprobe_syscall.c already does for its own CLONE_VM child. [1] https://github.com/llvm/llvm-project/pull/180943 Fixes: d2520bdb1932f ("selftests/bpf: Add uprobe multi pid filter test for clone-ed processes") Signed-off-by: KaFai Wan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261018005348.2195130-5-kafai.wan@linux.dev --- tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index f0baf5738b75..7fbcfddc3cc3 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c @@ -94,14 +94,14 @@ static int child_func(void *arg) /* wait for parent's kick */ err = read(child->go[0], &c, 1); if (err != 1) - exit(err); + _exit(err); uprobe_multi_func_1(); uprobe_multi_func_2(); uprobe_multi_func_3(); usdt_trigger(); - exit(errno); + _exit(errno); } static int spawn_child_flag(struct child *child, bool clone_vm) From a898ff0f3e9968b5f7c248d2af3cf1ddaf239e51 Mon Sep 17 00:00:00 2001 From: Masoud Aghasi Date: Thu, 8 Oct 2026 13:07:32 +0100 Subject: [PATCH 611/660] bpf: Fix incorrect handling of user flags in bpf_percpu_array_update For BPF_MAP_TYPE_PERCPU_ARRAY map, bpf_percpu_array_update() is not considering the possibility of a combination of (BPF_NOEXIST, BPF_EXIST) flags with (BPF_F_CPU, BPF_F_ALL_CPUS) flags. This causes the (BPF_NOEXIST, BPF_EXIST) flags to lose their effect in some cases. For example, using the (BPF_F_ALL_CPUS | BPF_EXIST) flag combination with bpf_map_update_elem() results in an incorrect EINVAL error response, even though the flag combination is valid. This patch fixes the bug by adding proper flag validations and checks. Before this patch, bpf_percpu_array_update() rejected BPF_F_ALL_CPUS | BPF_EXIST and BPF_F_ALL_CPUS | BPF_NOEXIST with -EINVAL. Also the BPF_F_CPU | BPF_NOEXIST were accepted. After the patch BPF_F_ALL_CPUS | BPF_EXIST is accepted and BPF_F_ALL_CPUS | BPF_NOEXIST and BPF_F_CPU | BPF_NOEXIST result in -EEXIST. This patch also starts returning -EINVAL for (BPF_EXIST | BPF_NOEXIST), with or without BPF_F_CPU, which was previously accepted. Fixes: 8eb76cb03f0f ("bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps") Signed-off-by: Masoud Aghasi Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Link: https://patch.msgid.link/20261008120734.1565972-2-maghasi@disroot.org --- include/linux/bpf.h | 3 +++ kernel/bpf/arraymap.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index feb7de6483e3..aeb8b9b97052 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4396,6 +4396,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0fe9afd4a591..133f189bf6d2 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -438,15 +438,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u32 size; int cpu, off = 0; - if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) - /* unknown flags */ + if (unlikely(((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS) || + ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; if (unlikely(index >= array->map.max_entries)) /* all elements were pre-allocated, cannot insert a new one */ return -E2BIG; - if (unlikely(map_flags == BPF_NOEXIST)) + if (unlikely(map_flags & BPF_NOEXIST)) /* all elements already exist */ return -EEXIST; From 4a9b25fe8d138f2e0e65b337ecf829ce7ce42d09 Mon Sep 17 00:00:00 2001 From: Masoud Aghasi Date: Thu, 8 Oct 2026 13:07:33 +0100 Subject: [PATCH 612/660] bpf: Fix incorrect handling of user flags by percpu hash map updates For BPF_MAP_TYPE_PERCPU_HASH and BPF_MAP_TYPE_LRU_PERCPU_HASH maps, htab_map_check_update_flags() and check_flags() are not considering the possibility of a combination of (BPF_NOEXIST, BPF_EXIST) flags with (BPF_F_CPU, BPF_F_ALL_CPUS) flags. This causes the (BPF_NOEXIST, BPF_EXIST) flags to lose their effect in some cases. For example, when using (BPF_F_CPU | BPF_EXIST) or (BPF_F_CPU | BPF_NOEXIST) flag combinations with bpf_map_update_elem() on a percpu hash map, the BPF_EXIST flag does not prevent new insertions as expected and BPF_NOEXIST flag does not prevent modification of existing entries as expected. This patch fixes the bug by adding proper flag validations and checks. Before this patch, htab_map_check_update_flags() rejected (BPF_F_ALL_CPUS | BPF_EXIST) and (BPF_F_ALL_CPUS | BPF_NOEXIST) with -EINVAL. After the patch those combinations are accepted. This patch also starts returning -EINVAL for (BPF_EXIST | BPF_NOEXIST), with or without BPF_F_CPU, which was previously accepted. Fixes: c6936161fd55 ("bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash and lru_percpu_hash maps") Signed-off-by: Masoud Aghasi Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Link: https://patch.msgid.link/20261008120734.1565972-3-maghasi@disroot.org --- kernel/bpf/hashtab.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 66ebb5a32807..bf78e09f4d51 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1367,11 +1367,11 @@ dec_count: static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, u64 map_flags) { - if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) + if (l_old && (map_flags & BPF_NOEXIST)) /* elem already exists */ return -EEXIST; - if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) + if (!l_old && (map_flags & BPF_EXIST)) /* elem doesn't exist, cannot update it */ return -ENOENT; @@ -1560,9 +1560,11 @@ err_lock_bucket: static int htab_map_check_update_flags(bool onallcpus, u64 map_flags) { + if (unlikely((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS)) + return -EINVAL; if (unlikely(!onallcpus && map_flags > BPF_EXIST)) return -EINVAL; - if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))) + if (unlikely(onallcpus && ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; return 0; } @@ -1661,7 +1663,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, * to remove older elem from htab and this removal * operation will need a bucket lock. */ - if (map_flags != BPF_EXIST) { + if (!(map_flags & BPF_EXIST)) { l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; From 22392799e251d8ddfbc766faf5c5fb0a5c69cf7d Mon Sep 17 00:00:00 2001 From: Masoud Aghasi Date: Thu, 8 Oct 2026 13:07:34 +0100 Subject: [PATCH 613/660] selftests/bpf: Add tests for percpu map flags combination All possible combinations of (BPF_EXIST, BPF_NOEXIST) and (BPF_F_ALL_CPUS, BPF_F_CPU) flags are covered for all percpu map types. As BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE does not support BPF_NOEXIST and also in order to reduce the amount of code duplicates, I added its new test scenarios to the existing cpu_flag_percpu_cgroup_storage test. But for other percpu map types, I added new tests to be able to exercise all flag combinations and the edge cases such as percpu LRU unnecessary deletion issue. Signed-off-by: Masoud Aghasi Signed-off-by: Alexei Starovoitov Acked-by: Leon Hwang Link: https://patch.msgid.link/20261008120734.1565972-4-maghasi@disroot.org --- .../selftests/bpf/prog_tests/percpu_alloc.c | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index 7b4a1e24363b..67124dd4d8a7 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -449,6 +449,95 @@ static void test_lru_percpu_hash_cpu_flag_create(void) test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); } +static bool verify_map_value_all_cpus(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem all_cpus")) + return false; + + for (i = 0 ; i < nr_cpus ; i++) { + if (!ASSERT_EQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value all_cpus")) + return false; + } + + return true; +} + +static bool verify_map_value_first_cpu(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem cpu")) + return false; + + if (!ASSERT_EQ(*(u32 *)values, expected_value, "bpf_map_lookup_elem value cpu")) + return false; + + for (i = 1 ; i < nr_cpus ; i++) { + if (!ASSERT_NEQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value cpu")) + return false; + } + + return true; +} + +static void test_percpu_cgroup_storage_flags_combination(struct bpf_map *map, int nr_cpus, + struct bpf_cgroup_storage_key *key) +{ + size_t value_sz = sizeof(u32), key_sz, elem_sz; + u32 *values, test_value = 0xAABBCCDD; + int err, map_fd; + u64 flags; + + key_sz = sizeof(*key); + elem_sz = roundup(value_sz, 8); + map_fd = bpf_map__fd(map); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map__update_elem(map, key, key_sz, values, elem_sz * nr_cpus, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, key, values, elem_sz, test_value, nr_cpus)) + goto out; + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem cpu|exist")) + goto out; + + verify_map_value_first_cpu(map_fd, key, values, elem_sz, test_value, nr_cpus); +out: + free(values); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -489,6 +578,7 @@ static void test_percpu_cgroup_storage_cpu_flag(void) goto out; test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); + test_percpu_cgroup_storage_flags_combination(map, nr_cpus, &key); out: bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); close(cgroup); @@ -537,6 +627,164 @@ static void test_hash_cpu_flag(void) test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); } +static void test_percpu_map_flags_combination(enum bpf_map_type map_type) +{ + u32 max_entries = 3, key = 0, first_value = 0xAABBCCDD, test_value = first_value; + size_t value_sz = sizeof(u32), elem_sz; + int err, map_fd, nr_cpus; + bool is_hash_map; + u32 *values; + u64 flags; + + is_hash_map = (map_type == BPF_MAP_TYPE_PERCPU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + elem_sz = roundup(value_sz, 8); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + map_fd = bpf_map_create(map_type, "test_flags_combination_map", + sizeof(u32), value_sz, max_entries, NULL); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) { + free(values); + return; + } + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map_update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + values[0] = test_value; + key = 0; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + values[0] = --test_value; + key = 1; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|noexist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = --test_value; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem all_cpus|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem cpu|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|exist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + if (map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) + goto out; + + /* Percpu LRU hash map should not delete old entries unnecessarily */ + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|exist")) + goto out; + + key = 0; + verify_map_value_all_cpus(map_fd, &key, values, elem_sz, first_value, nr_cpus); + +out: + close(map_fd); + free(values); +} + +static bool pin_current_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int err, cpu; + + err = sched_getaffinity(0, sizeof(*old_mask), old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + return false; + + cpu = sched_getcpu(); + if (!ASSERT_GE(cpu, 0, "sched_getcpu")) + return false; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + return ASSERT_OK(err, "sched_setaffinity"); +} + +static void test_percpu_hash_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_HASH); +} + +static void test_lru_percpu_hash_flags_combination(void) +{ + cpu_set_t old_mask; + + if (!pin_current_cpu(&old_mask)) + return; + + test_percpu_map_flags_combination(BPF_MAP_TYPE_LRU_PERCPU_HASH); + + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} + +static void test_percpu_array_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_ARRAY); +} + void test_percpu_alloc(void) { if (test__start_subtest("array")) @@ -565,4 +813,10 @@ void test_percpu_alloc(void) test_array_cpu_flag(); if (test__start_subtest("cpu_flag_hash")) test_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_combination_percpu_hash")) + test_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_lru_percpu_hash")) + test_lru_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_percpu_array")) + test_percpu_array_flags_combination(); } From 0f9b1e9ef584522b645a97eff8d1c8d0dcab3afb Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Fri, 9 Oct 2026 14:26:37 +0800 Subject: [PATCH 614/660] selftests/bpf: Add bpf_proactive_reclaim test bpf_proactive_reclaim() performs one bounded reclaim pass per call and, unlike a write to memory.reclaim, does not retry until the goal is reached. Charge 32 MiB of page cache to a cgroup, ask for all of it in one call, and check that the result is positive but well below the request: one pass is capped at MEMCG_CHARGE_BATCH pages, far short of 32 MiB on both 4K and 64K page kernels. The test deliberately covers only the kfunc itself. A full example of the intended asynchronous use, where a BPF program watches one cgroup's workingset refaults and reclaims another one from bpf_wq callbacks, is maintained out of tree at [1], released under GPLv2. Add CONFIG_MEMCG to the config fragment, without which mm/bpf_memcontrol.c is not built at all. [1] https://github.com/teawater/memcg-async-reclaim Signed-off-by: Hui Zhu Reviewed-by: Barry Song Link: https://lore.kernel.org/bpf/20261009062637.176005-1-hui.zhu@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/config | 1 + .../bpf/prog_tests/memcg_proactive_reclaim.c | 130 ++++++++++++++++++ .../bpf/progs/memcg_proactive_reclaim.c | 34 +++++ 3 files changed, 165 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c create mode 100644 tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index b3d4cf5987a9..49dfe939467c 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -58,6 +58,7 @@ CONFIG_LIRC=y CONFIG_LIVEPATCH=y CONFIG_LWTUNNEL=y CONFIG_LWTUNNEL_BPF=y +CONFIG_MEMCG=y CONFIG_MODULE_SIG=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_UNLOAD=y diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c new file mode 100644 index 000000000000..c22dcfabe856 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include "cgroup_helpers.h" +#include "memcg_proactive_reclaim.skel.h" + +#define CG_PATH "/memcg_proactive_reclaim" + +/* + * Large enough that a single reclaim pass cannot come close to it, so that + * "reclaimed less than was asked for" is not page-size noise: one pass is + * capped at MEMCG_CHARGE_BATCH pages, which is 256 KiB on 4K pages but 4 MiB + * on 64K pages. + */ +#define FILE_SIZE (32 * 1024 * 1024UL) +#define BUF_SIZE (64 * 1024) + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* + * The data file has to sit on a writable regular filesystem: tmpfs pages are + * charged as shmem, so whether they can be reclaimed at all depends on swap + * being available, and a read-only mount cannot host the file at all (EROFS + * applies to root too). /tmp is tmpfs on many systems, and test_progs is + * routinely run from a tmpfs working directory, so both candidates need the + * check. + */ +static const char *workload_dir(void) +{ + static const char * const dirs[] = { "/tmp", "." }; + struct statfs st; + int i; + + for (i = 0; i < ARRAY_SIZE(dirs); i++) + if (!statfs(dirs[i], &st) && st.f_type != TMPFS_MAGIC && + st.f_type != RAMFS_MAGIC && !access(dirs[i], W_OK)) + return dirs[i]; + + return NULL; +} + +void test_memcg_proactive_reclaim(void) +{ + struct memcg_proactive_reclaim *skel = NULL; + struct reclaim_args args = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &args, + .ctx_size_in = sizeof(args)); + + char data_file[PATH_MAX]; + static char buf[BUF_SIZE]; + __u64 cgroup_id; + const char *dir; + off_t off; + int cg_fd = -1, data_fd = -1, err; + + /* + * Both candidate directories can be unusable: tmpfs (initramfs-based + * CI, for instance) or a read-only mount. That is an environment + * limitation, not a test failure. + */ + dir = workload_dir(); + if (!dir) { + test__skip(); + return; + } + + snprintf(data_file, sizeof(data_file), + "%s/memcg_proactive_reclaim_XXXXXX", dir); + data_fd = mkstemp(data_file); + if (!ASSERT_GE(data_fd, 0, "mkstemp")) + return; + + cg_fd = cgroup_setup_and_join(CG_PATH); + if (!ASSERT_OK_FD(cg_fd, "cgroup_setup_and_join")) + goto out; + + cgroup_id = get_cgroup_id(CG_PATH); + if (!ASSERT_GT(cgroup_id, 0, "get_cgroup_id")) + goto out; + + skel = memcg_proactive_reclaim__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + /* Charge FILE_SIZE of clean page cache to the cgroup. */ + for (off = 0; off < (off_t)FILE_SIZE; off += sizeof(buf)) + if (!ASSERT_EQ(write(data_fd, buf, sizeof(buf)), + (ssize_t)sizeof(buf), "write")) + goto out; + if (!ASSERT_OK(fsync(data_fd), "fsync")) + goto out; + + args.cgroup_id = cgroup_id; + args.size = FILE_SIZE; + skel->bss->reclaimed = 0; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.memcg_proactive_reclaim), + &opts); + if (!ASSERT_OK(err, "test_run")) + goto out; + if (!ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + /* + * A single call is a single bounded pass: it reclaims something, but + * stops well short of the requested size instead of retrying until the + * goal is reached the way a write to memory.reclaim does. + */ + ASSERT_GT(skel->bss->reclaimed, 0, "reclaimed"); + ASSERT_LT(skel->bss->reclaimed, (__s64)FILE_SIZE, "single pass"); + +out: + if (skel) + memcg_proactive_reclaim__destroy(skel); + if (cg_fd >= 0) + close(cg_fd); + if (data_fd >= 0) { + close(data_fd); + unlink(data_file); + } + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c new file mode 100644 index 000000000000..b551192526fc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* Signed, because bpf_test_run_opts.retval is a __u32. */ +__s64 reclaimed; + +SEC("syscall") +int memcg_proactive_reclaim(struct reclaim_args *ctx) +{ + struct mem_cgroup *memcg; + struct cgroup *cgrp; + + cgrp = bpf_cgroup_from_id(ctx->cgroup_id); + if (!cgrp) + return 0; + + memcg = bpf_get_mem_cgroup(&cgrp->self); + if (memcg) { + reclaimed = bpf_proactive_reclaim(memcg, ctx->size, -1); + bpf_put_mem_cgroup(memcg); + } + bpf_cgroup_release(cgrp); + + return 0; +} + +char _license[] SEC("license") = "GPL"; From c3a0bfddaee4e1e5fce7f4426004669c017cd8f1 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Wed, 7 Oct 2026 16:25:29 +0000 Subject: [PATCH 615/660] bpf: Enforce trusted BTF ID argument provenance A global subprogram parameter tagged __arg_trusted is verified with PTR_TRUSTED, but the common call-site type check accepts bare PTR_TO_BTF_ID and MEM_RCU registers. The callee can consequently pass a pointer with weaker provenance to a kfunc that does not accept RCU-protected pointers. RCU protection only guarantees that the object remains allocated through a grace period. It does not guarantee that an object's refcount is nonzero, so an RCU-protected pointer cannot satisfy the trusted-pointer contract. For example, an RCU-protected bpf_cpumask loaded from a map may have a zero refcount. Passing it through __arg_trusted lets the callee pass it to bpf_cpumask_acquire(), whose plain refcount_inc() can retain an object that will be freed after the grace period. Make the existing provenance check contract-driven. Mark generated BTF ID kfunc argument types with PTR_TRUSTED, and add MEM_RCU only for KF_RCU kfuncs. Normalize PTR_MAYBE_NULL when the argument contract permits it. Then use the argument flags to require trusted or referenced provenance and allow MEM_RCU only when the contract says so. Split is_trusted_reg() so the normalized type is used for type and modifier checks while reference lookup still uses the original register. Conversion of scalar-struct BTF ID arguments to fixed-size memory continues to construct a new ARG_PTR_TO_MEM type without PTR_TRUSTED. Fixes: e2b3c4ff5d18 ("bpf: add __arg_trusted global func arg tag") Suggested-by: Amery Hung Signed-off-by: Yiyang Chen Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20261007-a3-arg-trusted-v4-v4-1-253ef3a1a69a@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 74e5ac248a10..5958425e9a2a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5501,15 +5501,17 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) +static bool is_trusted_reg_type(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) { /* A referenced register is always trusted. */ if (reg_is_referenced(env, reg)) return true; /* Types listed in the reg2btf_ids are always trusted */ - if (reg2btf_ids[base_type(reg->type)] && - !bpf_type_has_unsafe_modifiers(reg->type)) + if (reg2btf_ids[base_type(type)] && + !bpf_type_has_unsafe_modifiers(type)) return true; /* If a register is not referenced, it is trusted if it has the @@ -5521,8 +5523,14 @@ static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_st * Eventually, we should make PTR_TRUSTED the single source of truth * for whether a register is trusted. */ - return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && - !bpf_type_has_unsafe_modifiers(reg->type); + return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS && + !bpf_type_has_unsafe_modifiers(type); +} + +static bool is_trusted_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return is_trusted_reg_type(env, reg, reg->type); } static bool is_rcu_reg(const struct bpf_reg_state *reg) @@ -9798,6 +9806,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p { const u32 *arg_btf_id = fn->arg_btf_id[arg]; const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; + enum bpf_reg_type reg_type; if (!meta->btf) { const struct bpf_reg_types *compatible; @@ -9825,8 +9834,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 p } } - if (is_kfunc(meta) && (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type))) { + reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL); + if ((arg_type & PTR_TRUSTED) && + (!is_trusted_reg_type(env, reg, reg_type) || + bpf_type_has_unsafe_modifiers(reg_type))) { if (!(arg_type & MEM_RCU)) { const char *actual_type, *arg_name, *expected_type; @@ -14023,6 +14034,8 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, else proto->arg_btf_id[arg] = ref_id_ptr; + arg_type |= PTR_TRUSTED; + /* * A KF_RCU kfunc accepts an RCU-protected pointer where it would * otherwise demand a referenced or trusted one. Other argument kinds From 71b6a960c73f2ccff0932ff70aaf066bc21a960c Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Wed, 7 Oct 2026 16:25:30 +0000 Subject: [PATCH 616/660] selftests/bpf: Cover trusted BTF ID argument provenance Extend verifier_global_ptr_args.c with calls that pass bare and RCU-protected task pointers to a global __arg_trusted parameter. Both calls must be rejected because neither pointer is referenced or trusted. Add paired test-module kfuncs that return and accept a nullable trusted task pointer. Passing the return value directly must succeed, covering PTR_TRUSTED | PTR_MAYBE_NULL handling in the common argument checker. Signed-off-by: Yiyang Chen Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20261007-a3-arg-trusted-v4-v4-2-253ef3a1a69a@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/progs/iters_testmod.c | 11 ++++++ .../bpf/progs/verifier_global_ptr_args.c | 35 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 11 ++++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 2 ++ 4 files changed, 59 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index f65cc9766633..145750bb64ef 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -46,6 +46,17 @@ int iter_next_trusted_or_null(const void *ctx) return 0; } +SEC("syscall") +__success +int trusted_nullable_kfunc(const void *ctx) +{ + struct task_struct *task; + + task = bpf_kfunc_trusted_null_test(); + bpf_kfunc_trusted_nullable_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_rcu(const void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index 03507eeae3cb..4c55b70df072 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -289,6 +289,41 @@ __weak int subprog_void_untrusted(void *p __arg_untrusted) return *(int *)p; } +__weak int subprog_trusted_bare(struct task_struct *task __arg_trusted) +{ + return task->pid; +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}} ('subprog_trusted_bare')") +int bare_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + struct task_struct *wakee; + + if (!cur) + return 0; + wakee = cur->last_wakee; + if (!wakee) + return 0; + return subprog_trusted_bare(wakee); +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}}") +int memrcu_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + + if (!cur) + return 0; + return subprog_trusted_task_nullable(cur->real_parent); +} + __weak int subprog_char_untrusted(char *p __arg_untrusted) { return *(int *)p; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 38edb4addc5b..613abd27be5d 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -350,6 +350,15 @@ __bpf_kfunc void bpf_kfunc_trusted_task_test(struct task_struct *ptr) { } +__bpf_kfunc struct task_struct *bpf_kfunc_trusted_null_test(void) +{ + return NULL; +} + +__bpf_kfunc void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) +{ +} + __bpf_kfunc void bpf_kfunc_trusted_num_test(int *ptr) { } @@ -957,6 +966,8 @@ BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_zero_offset_test, KF_ACQUIRE) BTF_ID_FLAGS(func, bpf_kfunc_nested_release_test, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_trusted_vma_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_task_test) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_null_test, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_nullable_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test) BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU) BTF_ID_FLAGS(func, bpf_kfunc_ret_rcu_test, KF_RET_NULL | KF_RCU_PROTECTED) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index a6a93c901416..8f2f8b8a26e3 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -353,6 +353,8 @@ int bpf_kfunc_st_ops_inc10(struct st_ops_args *args) __ksym; void bpf_kfunc_trusted_vma_test(struct vm_area_struct *ptr) __ksym; void bpf_kfunc_trusted_task_test(struct task_struct *ptr) __ksym; +struct task_struct *bpf_kfunc_trusted_null_test(void) __ksym; +void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) __ksym; void bpf_kfunc_trusted_num_test(int *ptr) __ksym; void bpf_kfunc_rcu_task_test(struct task_struct *ptr) __ksym; struct task_struct *bpf_kfunc_ret_rcu_test(void) __ksym; From e2c978ad415ce223e71f6ddaa92778225eca0bef Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Fri, 9 Oct 2026 09:49:32 +0800 Subject: [PATCH 617/660] selftests/bpf: Run the cpumask map insert tests only for the test task verify_success() attaches each cpumask test program to task_newtask, forks a single child, and expects err to be 0 afterwards. Most programs return right away for other tasks than the test one. test_insert_leave and test_insert_remove_release don't, and insert a cpumask into the single entry of __cpumask_map for every task created in the system while they are attached. test_insert_leave leaves its cpumask in the map, so a second task created, for instance by a test running in parallel, fails to insert one and sets err to 3. Two tasks created at the same time can do the same to test_insert_remove_release. Return early for other tasks than the test one in both programs, as the other programs do. Fixes: 7b6abcfa15cd ("selftests/bpf: Add selftest suite for cpumask kfuncs") Signed-off-by: Qiliang Yuan Link: https://lore.kernel.org/bpf/20261009-selftests-bpf-cpumask-insert-filter-v1-1-3249218594d7@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/cpumask_success.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 36f77b9732d4..ddaba4318c57 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -460,6 +460,9 @@ int BPF_PROG(test_insert_leave, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *cpumask; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0; @@ -476,6 +479,9 @@ int BPF_PROG(test_insert_remove_release, struct task_struct *task, u64 clone_fla struct bpf_cpumask *cpumask; struct __cpumask_map_value *v; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0; From 4f842631f178f41baa2d2d444a277d54c07666a1 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 19 Oct 2026 01:13:27 +0000 Subject: [PATCH 618/660] bpf: Refuse the prologue for an extension if callers keep packet pointers An extension that writes to the packet of a tc, lwt_xmit or sk_skb program starts with bpf_unclone_prologue(). If the skb is cloned, that gives it a new head. It does not count as changing packet data. changes_pkt_data goes by the helpers and kfuncs that a program calls, and whether it gets a prologue is known only after it has been compared with its target. So the extension can replace a global function that leaves the packet alone, and the callers of that function keep their packet pointers across the call. These point into the old head, which is left to the other clone. Once that is freed, by the reader of a packet socket for example: BUG: KASAN: slab-use-after-free in bpf_prog_53bc21b9810cadbc_t_read+0x2e8/0x36e Read of size 1 at addr ffff888107bfe73c by task tt11/102 Freed by task 112: kfree+0x161/0x500 skb_release_data+0x2dc/0x340 consume_skb+0xa6/0x1c0 packet_recvmsg+0x4b3/0xa20 Nothing is wrong if the target program writes to the packet too: its own prologue has uncloned the skb, and that of the extension does nothing. Nor if the callers hold no packet pointers, like a dispatcher, or if the main function is replaced. Note for every global function that leaves the packet alone if it is called while packet pointers, or slices of the dynptr of a packet, are held. What the callers of a global function hold is not known while it is verified, so the functions that it calls are taken for such as well. Refuse an extension with a prologue for such a function, unless the target program has a prologue, both when the extension is loaded and when it is attached to another target. Functions can share a type id in func_info. The first of them is the one that is replaced, so look it up in one place. Fixes: 7e40781cc8b7 ("bpf: verifier: Use target program's type for access verifications") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261019011328.1843764-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 2 ++ include/linux/bpf_verifier.h | 2 ++ kernel/bpf/fixups.c | 6 ++++ kernel/bpf/verifier.c | 67 ++++++++++++++++++++++++++++++++---- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index aeb8b9b97052..dae33b1a7427 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1757,6 +1757,7 @@ struct bpf_func_info_aux { bool unreliable; bool called[BPF_FUNC_CTX_CNT]; bool verified[BPF_FUNC_CTX_CNT]; + bool called_with_pkt_ptrs; }; enum bpf_jit_poke_reason { @@ -1894,6 +1895,7 @@ struct bpf_prog_aux { bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; + bool has_prologue; bool might_sleep; bool kprobe_write_ctx; struct { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7a72bb06fe56..951a90458d18 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1381,6 +1381,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 713f14d73b06..bfbb0cf5c0a8 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -940,6 +940,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) "Extension writing packet data must take ctx and scalars only\n"); return -EINVAL; } + if (env->prog->type == BPF_PROG_TYPE_EXT && + bpf_check_ext_prologue(&env->log, + env->prog->aux->dst_prog, + env->prog->aux->attach_btf_id)) + return -EINVAL; + env->prog->aux->has_prologue = true; new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, BPF_PATCH_MOVE_TARGET); if (!new_prog) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5958425e9a2a..d13868846823 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10878,6 +10878,20 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) })); } +static bool holds_pkt_pointers(struct bpf_verifier_env *env) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + bool ret = false; + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + ret = true; + })); + + return ret; +} + enum { AT_PKT_END = -1, BEYOND_PKT_END = -2, @@ -11483,6 +11497,13 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); + /* + * What the callers of a global function hold is not known + * while it is verified. + */ + if (!env->subprog_info[subprog].changes_pkt_data && + (state->frame[0]->subprogno || holds_pkt_pointers(env))) + sub_aux->called_with_pkt_ptrs = true; /* * A non-void global function returns a 64-bit SCALAR_VALUE in @@ -22222,6 +22243,41 @@ static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf * return -EINVAL; } +/* Functions may share a type id. The first of them is attached to. */ +static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int i; + + for (i = 0; i < aux->func_info_cnt; i++) + if (aux->func_info[i].type_id == btf_id) + return i; + + return -1; +} + +/* + * The prologue of an extension gives a cloned skb a new head, unless the + * prologue of the target program has done that. The callers of the function + * that it replaces must not hold on to the old head. + */ +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int subprog = find_tgt_subprog(tgt_prog, btf_id); + + if (aux->has_prologue) + return 0; + + if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs) + return 0; + + bpf_log(log, + "Extension writing packet data cannot replace a function whose callers keep packet pointers\n"); + return -EINVAL; +} + /* * An extension is verified as a program that runs under the RCU read lock. Does * the function that it is to replace? @@ -22248,7 +22304,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, char trace_symbol[KSYM_SYMBOL_LEN]; const char prefix[] = "btf_trace_"; struct bpf_raw_event_map *btp; - int ret = 0, subprog = -1, i; + int ret = 0, subprog = -1; const struct btf_type *t; bool conservative = true; const char *tname, *fname; @@ -22287,11 +22343,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EINVAL; } - for (i = 0; i < aux->func_info_cnt; i++) - if (aux->func_info[i].type_id == btf_id) { - subprog = i; - break; - } + subprog = find_tgt_subprog(tgt_prog, btf_id); if (subprog == -1) { bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; @@ -22347,6 +22399,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program changes packet data, while original does not\n"); return -EINVAL; } + if (prog->aux->has_prologue && + bpf_check_ext_prologue(log, tgt_prog, btf_id)) + return -EINVAL; tgt_might_sleep = aux->func ? aux->func[subprog]->aux->might_sleep From 25bb03f0a47b622b9025a1558bf2095ec1f55d26 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 19 Oct 2026 16:30:05 +0800 Subject: [PATCH 619/660] selftests/bpf: Drop subtest descriptions in jit_insns_arena Drop the redundant __description macros from arena_zero_off and cast_to_user inside jit_insns_arena. This allows the test_loader framework to automatically fallback to the snake_case C function names rather than the description string for test display. Before: $ sudo ./test_progs -t jit_insns/arena_zero_off | tail -3 #191/1 jit_insns/arena access with zero offset:FAIL #191 jit_insns:FAIL Summary: 1/0 PASSED, 0 SKIPPED, 1/1 FAILED $ sudo ./test_progs -t jit_insns/cast_to_user #191/2 jit_insns/cast to a user address:OK #191 jit_insns:OK #192 jit_insns_blinded:OK Summary: 2/1 PASSED, 0 SKIPPED, 0/0 FAILED After: $ sudo ./test_progs -t jit_insns/arena_zero_off | tail -3 #191/1 jit_insns/arena_zero_off:FAIL #191 jit_insns:FAIL Summary: 1/0 PASSED, 0 SKIPPED, 1/1 FAILED $ sudo ./test_progs -t jit_insns/cast_to_user #191/2 jit_insns/cast_to_user:OK #191 jit_insns:OK #192 jit_insns_blinded:OK Summary: 2/1 PASSED, 0 SKIPPED, 0/0 FAILED The main goal is that if a test fails, developers can immediately recognize the exact, actionable test name from the standard output and rerun it without any confusion. Signed-off-by: Tiezhu Yang Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261019083005.12236-1-yangtiezhu@loongson.cn --- tools/testing/selftests/bpf/progs/jit_insns_arena.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/jit_insns_arena.c b/tools/testing/selftests/bpf/progs/jit_insns_arena.c index b396d5aefb5a..95736eae2a58 100644 --- a/tools/testing/selftests/bpf/progs/jit_insns_arena.c +++ b/tools/testing/selftests/bpf/progs/jit_insns_arena.c @@ -34,7 +34,6 @@ struct { #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) SEC("syscall") -__description("arena access with zero offset") __success __retval(0) int arena_zero_off(void *ctx) { @@ -82,7 +81,6 @@ int arena_zero_off(void *ctx) } SEC("syscall") -__description("cast to a user address") __success __retval(0) int cast_to_user(void *ctx) { From 55f17779d649ad0c769d154d649e8e6585ae23d7 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:43 +0200 Subject: [PATCH 620/660] bpf: Check exclusive maps once the program digest is final An exclusive map is bound to the digest of its program, which the verifier compared against prog->digest in check_map_prog_compatibility() as the map was added to the program. For a map an insn refers to that happens in check_and_resolve_insns(), after the digest is computed at the top of that function. For a map bound through a continuous fd_array it happens in process_fd_array(), which runs earlier, so for an unsigned program the comparison saw a zeroed digest and the load failed with -EACCES, "program's hash doesn't match map's excl_prog_hash", for the very program the map is exclusive to. Computing the digest earlier would not do: it has to cover the insns as the kernel verifies and runs them, after the CO-RE relocations it applies, or a loader could submit the insns of the exclusive program together with relocations that change them and pass the check with a different program. Leave the digest where it is and move the comparison instead: check every used map against the digest at the end of check_and_resolve_insns(), whether an insn referred to it or the array bound it. The claims a program makes on its maps move behind the check as well: bpf_map_claim(), which records the program as the user of the map for resolve_func_ptrs(), and the takeover of an insn array by bpf_insn_array_init(). Both ran as the map was added, so a program that is not the owner of an exclusive map could have flipped the owner's single-user state to shared, which a failed load does not undo, or held the owner's insn array while it was being verified. Now they run once every map has passed the check, and a rejected program leaves the maps as it found them. Nothing consults a claim before resolve_func_ptrs() and the fixup passes, which run later. The early digest of a signed program goes with the early comparison. Commit b707068e0ed9 ("bpf: Verify signed loader metadata at load time") computed it only so that the maps of a signed loader passed the check in process_fd_array(); the signature itself is verified over the submitted insns and the frozen maps by bpf_prog_verify_signature(), which does not read the digest. The maps of a signed program are now checked against the digest of the final insns like everyone else's, which covers the relocations the kernel applies and the signature does not; a loader program carries none. Fixes: baefdbdf6812 ("bpf: Implement exclusive map creation") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-2-memxor@gmail.com --- kernel/bpf/verifier.c | 85 ++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d13868846823..ca5301fe3f66 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20796,12 +20796,6 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, { enum bpf_prog_type prog_type = resolve_prog_type(prog); - if (map->excl_prog_sha && - memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { - verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); - return -EACCES; - } - if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { @@ -20899,6 +20893,61 @@ static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) } } +/* + * An exclusive map is bound to the digest of its program. The digest covers + * the insns as they are verified, after the CO-RE relocations the kernel + * applies, so it exists only once check_and_resolve_insns() has computed it, + * and the maps are checked against it there, whether an insn referred to + * them or a continuous fd_array bound them before. + */ +static int check_excl_maps(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + if (map->excl_prog_sha && + memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { + verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); + return -EACCES; + } + } + return 0; +} + +/* + * The program may use every map it referred to, see check_excl_maps(). Only now + * does it leave a trace on them: it claims each as its user, see + * bpf_map_claim(), and takes over its insn arrays. A program rejected before + * this point, one that is not the owner of an exclusive map in particular, + * leaves the maps as it found them. Nothing consults a claim before + * resolve_func_ptrs() and the fixup passes, which run later. + */ +static int claim_used_maps(struct bpf_verifier_env *env) +{ + int i, err; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + err = bpf_map_claim(env, map); + if (err) + return err; + + if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) + continue; + err = bpf_insn_array_init(map, env->prog); + if (err) { + verbose(env, "Failed to properly initialize insn array\n"); + return err; + } + env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; + } + return 0; +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -20936,20 +20985,6 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; - err = bpf_map_claim(env, map); - if (err) - return err; - - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - err = bpf_insn_array_init(map, env->prog); - if (err) { - verbose(env, "Failed to properly initialize insn array\n"); - return err; - } - env->insn_array_maps[env->insn_array_map_cnt++] = map; - env->prog->jit_required = true; - } - return env->used_map_cnt - 1; } @@ -21353,7 +21388,10 @@ next_insn: * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ - return 0; + err = check_excl_maps(env); + if (err) + return err; + return claim_used_maps(env); } /* that many pointers to functions are recognized in the maps of a program at most */ @@ -23619,11 +23657,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = -EINVAL; goto err_prep; } - if (env->signature) { - ret = bpf_prog_calc_tag(env->prog); - if (ret < 0) - goto err_prep; - } ret = process_fd_array(env, attr, uattr); if (ret) From c2e56c88f14ae7d0493f94e44b4bba9cb535fe81 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:44 +0200 Subject: [PATCH 621/660] selftests/bpf: Bind an exclusive map through a continuous fd_array Load the exclusive program of map_excl with the map in a continuous fd_array rather than through its insns alone, which has to work, and the other program the same way, which has to be refused. The two programs used to be identical and so shared a digest; give the second one a value of its own so that the kernel can tell them apart. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-3-memxor@gmail.com --- .../selftests/bpf/prog_tests/map_excl.c | 37 +++++++++++++++++++ tools/testing/selftests/bpf/progs/map_excl.c | 3 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 3f4422b9ffa6..9ff4cd3de2ec 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c @@ -161,12 +161,49 @@ static void test_map_excl_create_validation(void) ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); } +/* + * The exclusive map bound through a continuous fd_array rather than by an + * insn. The kernel resolves the array before it verifies the program, so it + * has to have the digest of the program at hand by then. + */ +static void test_map_excl_fd_array(void) +{ + struct map_excl *skel = map_excl__open(); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int err, fd, map_fd; + + if (!ASSERT_OK_PTR(skel, "map_excl__open")) + return; + err = bpf_map__set_exclusive_program(skel->maps.excl_map, skel->progs.should_have_access); + if (!ASSERT_OK(err, "bpf_map__set_exclusive_program")) + goto out; + + bpf_program__set_autoload(skel->progs.should_have_access, true); + bpf_program__set_autoload(skel->progs.should_not_have_access, true); + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "map_excl prepare")) + goto out; + + map_fd = bpf_map__fd(skel->maps.excl_map); + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + fd = bpf_program__clone(skel->progs.should_have_access, &opts); + if (ASSERT_GE(fd, 0, "exclusive program with the map in fd_array")) + close(fd); + fd = bpf_program__clone(skel->progs.should_not_have_access, &opts); + ASSERT_EQ(fd, -EACCES, "other program with the map in fd_array"); +out: + map_excl__destroy(skel); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_fd_array")) + test_map_excl_fd_array(); if (test__start_subtest("map_excl_no_map_in_map")) test_map_excl_no_map_in_map(); if (test__start_subtest("map_excl_no_map_iter")) diff --git a/tools/testing/selftests/bpf/progs/map_excl.c b/tools/testing/selftests/bpf/progs/map_excl.c index d461684728e4..90f127073dff 100644 --- a/tools/testing/selftests/bpf/progs/map_excl.c +++ b/tools/testing/selftests/bpf/progs/map_excl.c @@ -27,7 +27,8 @@ int should_have_access(void *ctx) SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") int should_not_have_access(void *ctx) { - int key = 0, value = 0xdeadbeef; + /* a different value, so that the two programs do not share a digest */ + int key = 0, value = 0xbaadf00d; bpf_map_update_elem(&excl_map, &key, &value, 0); return 0; From b1eaee8b9a95897a5df843c623aa55d9585cd9d8 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:45 +0200 Subject: [PATCH 622/660] libbpf: Keep the fd_array fill index out of bpf_program__clone() bpf_program__clone() hands the kernel the fd_array of the object along with obj->fd_array_cnt when the caller supplies neither. That counter is not a count the kernel can use: it is the index of the next free slot of an array built for the sparse fd_array API, where module BTF fds start at slot 1 and slot 0 stays empty, since a kfunc insn with offset 0 names vmlinux BTF rather than a slot. A non-zero fd_array_cnt turns the array into a continuous one, which the kernel resolves slot by slot, and slot 0 holds 0. Cloning a program from an object that calls a module kfunc thus fails with "fd 0 is not pointing to valid bpf_map or btf", for every program of the object, which is what veristat runs into on such an object. bpf_object__load() passes the array without a count. Do the same in bpf_program__clone(), leaving the count to callers who pass an array of their own, and rename the counter to fd_array_used so that it no longer reads like the uapi field. Fixes: 970bd2dced35 ("libbpf: Introduce bpf_program__clone()") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Mykyta Yatsenko Link: https://patch.msgid.link/20261019132203.3034698-4-memxor@gmail.com --- tools/lib/bpf/libbpf.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cb09ded90773..fcedb78abe00 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -794,7 +794,12 @@ struct bpf_object { int *fd_array; size_t fd_array_cap; - size_t fd_array_cnt; + /* + * Slots handed out so far, slot 0 included. A kfunc insn names a module + * BTF slot through its offset, slot 0 stands for vmlinux BTF and stays + * empty, so the array goes to the kernel without a count. + */ + size_t fd_array_used; struct usdt_manager *usdt_man; @@ -9716,22 +9721,22 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, /* set index for module BTF fd in fd_array, if unset */ if (mod_btf && !mod_btf->fd_array_idx) { /* insn->off is s16 */ - if (obj->fd_array_cnt == INT16_MAX) { + if (obj->fd_array_used == INT16_MAX) { pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", ext->name, mod_btf->fd_array_idx); return -E2BIG; } /* Cannot use index 0 for module BTF fd */ - if (!obj->fd_array_cnt) - obj->fd_array_cnt = 1; + if (!obj->fd_array_used) + obj->fd_array_used = 1; ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), - obj->fd_array_cnt + 1); + obj->fd_array_used + 1); if (ret) return ret; - mod_btf->fd_array_idx = obj->fd_array_cnt; + mod_btf->fd_array_idx = obj->fd_array_used; /* we assume module BTF FD is always >0 */ - obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; + obj->fd_array[obj->fd_array_used++] = mod_btf->fd; } ext->is_set = true; @@ -10940,7 +10945,7 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex; attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; - attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt; + attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; From d837f008bfe6ee5511ad908e0e939e197b62f580 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:46 +0200 Subject: [PATCH 623/660] selftests/bpf: Clone a program that calls a module kfunc Prepare an object whose program calls a kfunc of bpf_testmod and load the program through bpf_program__clone(), the way veristat loads programs. The module BTF fd sits in slot 1 of the fd_array of the object and slot 0 is empty; the clone used to pass the fill index of that array as fd_array_cnt, which made the kernel resolve slot 0 and fail the load with "fd 0 is not pointing to valid bpf_map or btf". Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-5-memxor@gmail.com --- .../bpf/prog_tests/clone_module_kfunc.c | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c diff --git a/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c new file mode 100644 index 000000000000..fa6ccbaa7fd9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "kfunc_call_test.skel.h" + +/* + * bpf_program__clone() loads a program from a prepared object with the + * fd_array of the object, where module BTF fds start at slot 1 and slot 0 is + * empty. It used to pass the fill index of that array as fd_array_cnt, which + * made the kernel resolve slot 0 and fail the load. + */ +void test_clone_module_kfunc(void) +{ + struct kfunc_call_test *skel; + int err, fd; + + skel = kfunc_call_test__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + + fd = bpf_program__clone(skel->progs.kfunc_call_test1, NULL); + if (ASSERT_GE(fd, 0, "clone")) + close(fd); +out: + kfunc_call_test__destroy(skel); +} From 10832993c9419719c656b2ddc93161faedd5fd57 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:47 +0200 Subject: [PATCH 624/660] bpf: Add the __arena and __nullable BTF type tags Kernel structures increasingly want to publish pointers into a BPF arena to the BPF programs they are shared with, and some pointer members that reach programs as trusted are legitimately NULL at times. Neither can be expressed today: a pointer member reaches the program as an ordinary kernel pointer, and sched_ext resorts to a plain u64 holding a pre-translated arena address that programs cast before use. Define __arena and __nullable as BTF type tags, the way __user, __rcu and __percpu already are, so that tagged members survive into vmlinux and module BTF and the verifier can act on them. To kernel code and to sparse the qualifiers are empty: the pointers are ordinary kernel pointers. The verifier side follows in the next patches. The tags live in a header of their own, bpf_compiler_types.h, rather than in compiler_types.h. They mean something to the BPF verifier only, and only the few structures shared with BPF programs that want to publish such a pointer carry them, so the code that declares such a structure includes the header and the rest of the kernel never sees the qualifiers. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-6-memxor@gmail.com --- include/linux/bpf_compiler_types.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 include/linux/bpf_compiler_types.h diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 000000000000..5c9e71acfdac --- /dev/null +++ b/include/linux/bpf_compiler_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */ From bc7c2824682be8dda2e0aa43b08aff822b8fca0d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:48 +0200 Subject: [PATCH 625/660] bpf: Recognize __nullable on pointer members of kernel structs A pointer member of a kernel struct that the verifier derives as trusted or RCU-protected is assumed to be non-NULL, so a load through it is a plain dereference. The BTF_TYPE_SAFE_*_OR_NULL lists exist for members known to be NULL at times, but they live in the verifier and have to be extended by hand. Teach btf_struct_walk() to report the "nullable" BTF type tag on a pointer member as PTR_MAYBE_NULL. The tag combines with any other tag on the member; it only says that the pointer may be NULL. The verifier keeps the flag where NULL would otherwise be assumed away: on trusted and RCU results, which a program dereferences as they are, and on per-CPU results, which the per-CPU helpers shift without a check. Such a pointer has to be tested before it is dereferenced or passed on. Plain and untrusted results are probed anyway and lose the flag, as the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either, so a tagged member of an existing struct asks no more of the programs that read it than a list entry does, and the tag can take the place of the lists. The next patch uses the tag on __arena members to decide whether the injected address translation has to preserve NULL. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-7-memxor@gmail.com --- kernel/bpf/btf.c | 18 ++++++++++++++++-- kernel/bpf/verifier.c | 10 ++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a9bf4c3c0a27..012b6c5c3eff 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3569,7 +3569,8 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } -bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +/* Whether the pointee of pointer type @t carries the type tag @name */ +static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) { if (!btf_type_is_ptr(t)) return false; @@ -3578,13 +3579,18 @@ bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) t = btf_type_by_id(btf, t->type)) { if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) continue; - if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) return true; } return false; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + return btf_ptr_has_type_tag(btf, t, "arena"); +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7973,6 +7979,14 @@ error: id = ctx.id; stype = ctx.t; + /* + * __nullable combines with any of the above: it only says + * that the pointer may be NULL. The caller decides whether + * that matters for the register type it derives. + */ + if (btf_ptr_has_type_tag(btf, mtype, "nullable")) + tmp_flag |= PTR_MAYBE_NULL; + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ca5301fe3f66..1679ffec431b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6754,6 +6754,16 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, clear_trusted_flags(&flag); } + /* + * A __nullable member only matters where NULL would otherwise be + * assumed away, which is anywhere but on a plain or an untrusted + * pointer: those are probed on a dereference, which tolerates NULL, and + * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A + * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. + */ + if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) + flag &= ~PTR_MAYBE_NULL; + if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) From 7efe1d4e11395e31c49122f0052b57f0513c7e83 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:49 +0200 Subject: [PATCH 626/660] bpf: Turn loads of __arena pointer members into arena pointers A kernel struct shared with BPF programs may want to point at arena memory, e.g. a sched_ext argument struct that the kernel builds in the scheduler's arena and that refers to another arena object. A pointer member reaches the program typed as a kernel pointer, so sched_ext publishes a plain u64 holding a pre-translated arena address that the program casts before use. Translate in the verifier instead. When a load through a kernel struct pointer lands on a member tagged __arena, btf_struct_walk() reports it and the destination register becomes PTR_TO_ARENA. The member holds the kernel address of the arena memory; the fixup pass appends the conversion to the arena pointer form that the struct_ops trampoline applies to __arena arguments, after a plain load or a load-acquire of the member: r1 = *(u64 *)(r2 + off) if r1 == 0 goto +1 // __nullable members and probed loads w1 -= lo32(kern_vm_start) A probed load, through a plain or an untrusted pointer, keeps the NULL test as well: it reads as 0 when it faults, and the subtraction must not turn that into an arena pointer. An acquire load only comes through a pointer that cannot fault. Stores to kernel structs are rejected as before, and with them the atomics that write such a member. A member inside a multi-member union is rejected too: an offset into a union resolves to its first member, so a program reading another member there would receive an arena pointer it did not ask for. Loads through program-allocated objects and program BTF are unaffected, there the tag is what the compiler emits for the arena address space. The rewrite is per insn while verification is per path, so the aux data records what an insn loaded on any path: an insn that loaded an arena pointer member on one path and a different member on another is rejected by the fixup pass, while paths disagreeing only on __nullable keep the NULL test, which a never-NULL member can afford. As for __arena arguments, the program needs an associated arena, which also supplies the base to subtract. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-8-memxor@gmail.com --- include/linux/bpf_verifier.h | 8 +++++ kernel/bpf/btf.c | 30 ++++++++++++++++ kernel/bpf/fixups.c | 66 +++++++++++++++++++++++++++++++++--- kernel/bpf/verifier.c | 53 +++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 951a90458d18..54481da4ed2d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -793,6 +793,14 @@ struct bpf_insn_aux_data { */ u64 fastcall_spills_num:3; u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; /* below flags are initialized once */ u64 jmp_point:1; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 012b6c5c3eff..538129ed2418 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7707,6 +7707,7 @@ enum bpf_struct_walk_result { WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; @@ -7987,6 +7988,30 @@ error: if (btf_ptr_has_type_tag(btf, mtype, "nullable")) tmp_flag |= PTR_MAYBE_NULL; + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -8089,6 +8114,11 @@ int btf_struct_access(struct bpf_verifier_log *log, *flag = MEM_RDONLY | PTR_UNTRUSTED; ret = PTR_TO_MEM; goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: ret = SCALAR_VALUE; goto check_variable_offset; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index bfbb0cf5c0a8..34b53ebb5841 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -882,6 +882,37 @@ static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -1026,6 +1057,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -1117,11 +1163,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1679ffec431b..015dc1bccebe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6585,6 +6585,45 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -6694,6 +6733,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -7475,6 +7517,17 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); From 9278bedb22deb16f5c7084be5a7791f62520b939 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:50 +0200 Subject: [PATCH 627/660] selftests/bpf: Test __nullable pointer members Add a struct_ops member to bpf_testmod whose argument struct carries two task pointers, one plain and one __nullable, both made trusted through a __safe_trusted alias of the struct in the module BTF, and a kfunc that calls the member once with the nullable pointer unset and once set. The callback acquires the plain pointer as is and the nullable one only after a check, driven through the kfunc from a syscall program. The failure case passes the nullable pointer to bpf_task_acquire() unchecked and is rejected as possibly NULL. More members cover the other kinds of pointer: a per-CPU one, which only the per-CPU helpers consume and which keeps the flag, the struct itself through a plain pointer and through an untrusted one out of a __kptr_untrusted map value that is never set, and a pointer to a scalar, which are probed and lose it. The callback uses each after a check; a failure test passes the per-CPU pointer unchecked, and three programs read the probed ones unchecked and load. bpf_testmod_ops3 gets the base helpers for the per-CPU helpers. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-9-memxor@gmail.com --- .../bpf/prog_tests/test_struct_ops_nullable.c | 62 ++++++++++++++ .../selftests/bpf/progs/struct_ops_nullable.c | 82 +++++++++++++++++++ .../bpf/progs/struct_ops_nullable_fail.c | 28 +++++++ .../progs/struct_ops_nullable_fail__percpu.c | 28 +++++++ .../progs/struct_ops_nullable_probed__plain.c | 26 ++++++ .../struct_ops_nullable_probed__scalar.c | 26 ++++++ .../struct_ops_nullable_probed__untrusted.c | 45 ++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 43 ++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.h | 34 ++++++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 1 + tools/testing/selftests/bpf/testing_helpers.c | 45 ++++++++++ tools/testing/selftests/bpf/testing_helpers.h | 7 ++ 12 files changed, 427 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c new file mode 100644 index 000000000000..028f070f2e5f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + +#include "struct_ops_nullable.skel.h" +#include "struct_ops_nullable_fail.skel.h" +#include "struct_ops_nullable_fail__percpu.skel.h" +#include "struct_ops_nullable_probed__plain.skel.h" +#include "struct_ops_nullable_probed__scalar.skel.h" +#include "struct_ops_nullable_probed__untrusted.skel.h" + +/* + * Attach a callback that gets a trusted and a trusted __nullable task pointer + * through its argument struct and drive it through the + * bpf_testmod_ops3_call_test_nullable() kfunc. + */ +static void nullable_member(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_nullable *skel; + struct bpf_link *link; + int err; + + skel = struct_ops_nullable__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_nullable__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_nullable); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +out: + struct_ops_nullable__destroy(skel); +} + +/* Serialized because it attaches the singleton bpf_testmod_ops3. */ +void serial_test_struct_ops_nullable(void) +{ + int has_tag; + + /* the tag reaches the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_nullable_args", "task_or_null", "nullable"); + if (!ASSERT_GE(has_tag, 0, "testmod_btf")) + return; + if (!has_tag) { + printf("%s:SKIP: no nullable type tag in the bpf_testmod BTF\n", __func__); + test__skip(); + return; + } + if (test__start_subtest("nullable_member")) + nullable_member(); + RUN_TESTS(struct_ops_nullable_fail); + RUN_TESTS(struct_ops_nullable_fail__percpu); + RUN_TESTS(struct_ops_nullable_probed__plain); + RUN_TESTS(struct_ops_nullable_probed__scalar); + RUN_TESTS(struct_ops_nullable_probed__untrusted); +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c new file mode 100644 index 000000000000..929969e1e012 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +/* an untrusted pointer to the argument struct, never set, for the verifier */ +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +SEC("struct_ops/test_nullable") +int BPF_PROG(test_nullable_cb, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + struct task_struct *t; + int ret = 0, *i, zero = 0; + + /* trusted, so good for a kfunc as is */ + t = bpf_task_acquire(args->task); + if (!t) + return -1; + bpf_task_release(t); + + /* trusted but __nullable, so a check has to come first */ + if (args->task_or_null) { + t = bpf_task_acquire(args->task_or_null); + if (!t) + return -1; + bpf_task_release(t); + ret = 1; + } + /* per-CPU and __nullable: a check has to come first, per_cpu_ptr() has none */ + if (args->pcpu_or_null) { + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + if (v->val != 0x5ca1ab1e) + return -1; + } + /* through a plain pointer the loads are probed, __nullable still asks for a check */ + t = args->self->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + /* a pointer to a scalar walks as untrusted memory, __nullable applies all the same */ + i = args->int_or_null; + if (i && *i != 7) + return -1; + /* through an untrusted pointer the member is probed too, and stays nullable */ + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (u) { + t = u->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + } + return ret; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable = { + .test_nullable = (void *)test_nullable_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ + /* one call without task_or_null, one with */ + return bpf_testmod_ops3_call_test_nullable() != 2; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c new file mode 100644 index 000000000000..c15b3a8c8d39 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* The member is trusted, but __nullable, so it needs a check before the kfunc. */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(unchecked_nullable, struct bpf_testmod_nullable_args *args) +{ + struct task_struct *t; + + t = bpf_task_acquire(args->task_or_null); + if (t) + bpf_task_release(t); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_nullable, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c new file mode 100644 index 000000000000..652015189162 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The member is per-CPU and __nullable: per_cpu_ptr() shifts NULL like any + * other pointer, so a check has to come before the helper. + */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(unchecked_percpu, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + return v->val; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_percpu, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c new file mode 100644 index 000000000000..c45f6d34f7d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * Through a plain pointer the member is probed, which tolerates NULL, so the + * tag asks for no check there, as the BTF_TYPE_SAFE_*_OR_NULL lists never did. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_plain, struct bpf_testmod_nullable_args *args) +{ + return args->self->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_plain, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c new file mode 100644 index 000000000000..ff768746b0da --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * A pointer to a scalar walks as untrusted memory, which is probed, so the + * tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_scalar, struct bpf_testmod_nullable_args *args) +{ + return *args->int_or_null; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_scalar, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c new file mode 100644 index 000000000000..961999139841 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +/* + * Through an untrusted pointer the member is probed, which tolerates NULL, so + * the tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_untrusted, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + int zero = 0; + + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (!u) + return 0; + return u->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_untrusted, +}; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 613abd27be5d..37d5824810d1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -506,6 +506,11 @@ static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair return 0; } +static int bpf_testmod_ops3__test_nullable(struct bpf_testmod_nullable_args *args) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, @@ -513,6 +518,7 @@ static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, .test_arena_stack = bpf_testmod_ops3__test_arena_stack, .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, + .test_nullable = bpf_testmod_ops3__test_nullable, }; static void bpf_testmod_test_struct_ops3(void) @@ -553,6 +559,41 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) return st_ops3->test_arena_multislot(p, ptr__arena); } +/* + * The members of struct bpf_testmod_nullable_args are trusted the way the + * verifier's BTF_TYPE_SAFE_TRUSTED() lists vmlinux members: through a struct + * of the same name with the __safe_trusted suffix in the BTF of the module. + */ +struct bpf_testmod_nullable_args__safe_trusted { + struct task_struct *task; + struct task_struct *task_or_null; +}; + +static DEFINE_PER_CPU(struct bpf_testmod_pcpu, bpf_testmod_nullable_pcpu); + +/* + * Call test_nullable() with the __nullable members unset, then set. The + * callback returns 1 when it found a task in @task_or_null, so the expected + * result is 2. + */ +__bpf_kfunc int bpf_testmod_ops3_call_test_nullable(void) +{ + struct bpf_testmod_nullable_args args = { .task = current }; + int seven = 7, ret; + + BTF_TYPE_EMIT(struct bpf_testmod_nullable_args__safe_trusted); + + if (!st_ops3 || !st_ops3->test_nullable) + return -EOPNOTSUPP; + args.self = &args; + ret = st_ops3->test_nullable(&args); + args.task_or_null = current; + this_cpu_write(bpf_testmod_nullable_pcpu.val, 0x5ca1ab1e); + args.pcpu_or_null = &bpf_testmod_nullable_pcpu; + args.int_or_null = &seven; + return ret | st_ops3->test_nullable(&args) << 1; +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -980,6 +1021,7 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_nullable) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) @@ -1956,6 +1998,7 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { }; static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = { + .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_testmod_ops_is_valid_access, }; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 2b12f81bf748..1a67564a6b8d 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -114,6 +114,38 @@ struct bpf_testmod_arena_pair { u64 b; }; +/* + * The qualifiers are BTF type tags for the verifier. The module gets them + * from bpf_compiler_types.h, user space and BPF programs need them empty. + */ +#ifdef __KERNEL__ +#include +#endif +#ifndef __nullable +#define __nullable +#endif +#ifndef __percpu +#define __percpu +#endif + +struct bpf_testmod_pcpu { + int val; +}; + +/* Arguments of test_nullable(), trusted through the __safe_trusted alias in the module */ +struct bpf_testmod_nullable_args { + /* never NULL */ + struct task_struct *task; + /* NULL on the first call, set on the second */ + struct task_struct __nullable *task_or_null; + /* NULL on the first call, a per-CPU variable on the second */ + struct bpf_testmod_pcpu __percpu __nullable *pcpu_or_null; + /* NULL on the first call, set on the second */ + int __nullable *int_or_null; + /* this struct again, through a plain pointer, which the verifier probes */ + struct bpf_testmod_nullable_args *self; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); @@ -125,6 +157,8 @@ struct bpf_testmod_ops3 { u64 g, u64 h, u64 *ptr); /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); + /* Used to test __nullable pointer members. */ + int (*test_nullable)(struct bpf_testmod_nullable_args *args); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 8f2f8b8a26e3..1e7b1f1e5148 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -224,6 +224,7 @@ int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_nullable(void) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 202c40e536b2..1e3e31ac36eb 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "disasm.h" #include "test_progs.h" @@ -644,3 +645,47 @@ out: } return -1; } + +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_member *m; + const struct btf_type *t; + int err = 0, id, i; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return -errno; + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + err = -errno; + goto out; + } + id = btf__find_by_name_kind_own(module_btf, struct_name, BTF_KIND_STRUCT); + if (id < 0) { + err = id; + goto out; + } + t = btf__type_by_id(module_btf, id); + err = -ENOENT; + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + if (strcmp(btf__name_by_offset(module_btf, m->name_off), member)) + continue; + err = 0; + /* the tag sits on the pointee, behind the pointer and other modifiers */ + for (id = m->type; id; id = t->type) { + t = btf__type_by_id(module_btf, id); + if (btf_is_type_tag(t) && !strcmp(btf__name_by_offset(module_btf, t->name_off), tag)) { + err = 1; + break; + } + if (!btf_is_ptr(t) && !btf_is_mod(t) && !btf_is_typedef(t)) + break; + } + break; + } +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + return err; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index d5083fb92ccd..36a75158495b 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -67,4 +67,11 @@ int stack_mprotect(void); /* Runs diff(1) on mismatch */ int compare_text_to_expected(const char *actual, const char *expected); +/* + * Whether @member of struct @struct_name in the BTF of bpf_testmod carries + * the BTF type tag @tag: 1 if so, 0 if not, negative on a BTF error or when + * the struct or the member is missing. + */ +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag); + #endif /* __TESTING_HELPERS_H */ From 1dad22de9c035afbe93b9c8f5eab24d25aaf6bb8 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:51 +0200 Subject: [PATCH 628/660] selftests/bpf: Test __arena pointer members Add a struct_ops member to bpf_testmod whose argument struct publishes arena pointers: a plain one, a __nullable one that is left NULL, one behind a nested struct, one inside a union, next to a scalar, together with a kfunc that fills the struct with the kernel address of an __arena argument and calls the member. Two callbacks read the pointers back. One shares the header with the module, so the members are arena pointers on the program side as well and need no cast; it also records the register a load produces, which must be the arena offset with the upper 32 bits clear, reads the member with a load-acquire as well, which has to yield the same pointer, and hands the pointer to an __arena kfunc. The other mirrors the vmlinux.h rendering of the struct, a kernel pointer with the type tag as an attribute, and casts the members to the arena address space the way a program built against vmlinux.h has to. Both increment the value through each pointer and the trigger checks the count. The first callback also reads a member through a plain pointer to a bad address, a probed load that is skipped or faults, and checks that it reads as NULL rather than as an arena pointer. The failure cases cover a store to the member, which a struct_ops program cannot do to any argument member, a load from the union, an exchange, one insn loading the arena pointer on one path and a scalar or a memory pointer on another, and a program without an arena. On a kernel whose module BTF carries no type tags the member tests are reported as skipped. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-10-memxor@gmail.com --- .../bpf/prog_tests/test_struct_ops_arena.c | 75 ++++++++++++ .../bpf/progs/struct_ops_arena_member.c | 109 ++++++++++++++++++ .../struct_ops_arena_member_fail__mismatch.c | 45 ++++++++ ...ruct_ops_arena_member_fail__mismatch_mem.c | 54 +++++++++ .../struct_ops_arena_member_fail__no_arena.c | 24 ++++ .../struct_ops_arena_member_fail__union.c | 34 ++++++ .../struct_ops_arena_member_fail__write.c | 36 ++++++ .../struct_ops_arena_member_fail__xchg.c | 40 +++++++ .../progs/struct_ops_arena_member_render.c | 85 ++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 24 ++++ .../selftests/bpf/test_kmods/bpf_testmod.h | 44 ++++++- .../bpf/test_kmods/bpf_testmod_kfunc.h | 1 + 12 files changed, 569 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 7f9f54ba3fbe..1c04473fb42d 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -5,6 +5,14 @@ #include "struct_ops_arena.skel.h" #include "struct_ops_arena_attach.skel.h" #include "struct_ops_arena_fail.skel.h" +#include "struct_ops_arena_member.skel.h" +#include "struct_ops_arena_member_render.skel.h" +#include "struct_ops_arena_member_fail__union.skel.h" +#include "struct_ops_arena_member_fail__xchg.skel.h" +#include "struct_ops_arena_member_fail__mismatch.skel.h" +#include "struct_ops_arena_member_fail__mismatch_mem.skel.h" +#include "struct_ops_arena_member_fail__write.skel.h" +#include "struct_ops_arena_member_fail__no_arena.skel.h" #if defined(__x86_64__) || defined(__aarch64__) /* @@ -101,6 +109,54 @@ static void arena_arg_attach(void) struct_ops_arena__destroy(skel); } + +/* Attach @st_ops and run @trigger once, it returns 0 when everything checked out. */ +static void attach_and_trigger(struct bpf_map *st_ops, struct bpf_program *trigger) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *link; + int err; + + link = bpf_map__attach_struct_ops(st_ops); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +} + +/* + * Attach a callback that reads arena pointers out of its argument struct. + * The header is shared with the module, so the members are arena pointers + * on the program side as well. + */ +static void arena_member(void) +{ + struct struct_ops_arena_member *skel; + + skel = struct_ops_arena_member__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member, skel->progs.trigger); + struct_ops_arena_member__destroy(skel); +} + +/* The same through the vmlinux.h rendering of the members, which takes casts. */ +static void arena_member_render(void) +{ + struct struct_ops_arena_member_render *skel; + + skel = struct_ops_arena_member_render__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member_render__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member_render, skel->progs.trigger); + struct_ops_arena_member_render__destroy(skel); +} #endif /* @@ -116,12 +172,31 @@ void serial_test_struct_ops_arena(void) * arena arguments", so the programs cannot even load. */ #if defined(__x86_64__) || defined(__aarch64__) + int has_tag; + if (test__start_subtest("arena_arg")) arena_arg(); if (test__start_subtest("arena_arg_fail")) arena_arg_fail(); if (test__start_subtest("arena_arg_attach")) arena_arg_attach(); + /* the tags reach the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_arena_member_args", "ptr", "arena"); + if (ASSERT_GE(has_tag, 0, "testmod_btf") && has_tag) { + if (test__start_subtest("arena_member")) + arena_member(); + if (test__start_subtest("arena_member_render")) + arena_member_render(); + RUN_TESTS(struct_ops_arena_member_fail__union); + RUN_TESTS(struct_ops_arena_member_fail__xchg); + RUN_TESTS(struct_ops_arena_member_fail__mismatch); + RUN_TESTS(struct_ops_arena_member_fail__mismatch_mem); + RUN_TESTS(struct_ops_arena_member_fail__write); + RUN_TESTS(struct_ops_arena_member_fail__no_arena); + } else if (test__start_subtest("arena_member")) { + /* so that the missing member tests show up as skipped */ + test__skip(); + } #else test__skip(); #endif diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c new file mode 100644 index 000000000000..a2218cd81619 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; +/* the register value a load of args->ptr produced in the callback */ +u64 __arena member_ptr_val; + +/* + * The header is shared with the module, so the members are u64 __arena * on + * this side too and the loads need no cast. The verifier derives the type of + * the loaded value from the module BTF, where the members carry the tag. + */ +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = args->ptr, *acq; + u64 raw; + + arena_touch++; + + /* the register right after the load, untouched by the compiler */ + asm volatile("%[raw] = *(u64 *)(%[args] + %[off])" + : [raw]"=r"(raw) + : [args]"r"(args), + [off]"i"(offsetof(struct bpf_testmod_arena_member_args, ptr))); + member_ptr_val = raw; + + *ptr += 1; + /* an acquire load of the member yields the same arena pointer */ + asm volatile("r1 = %[args];\n" + ".8byte 0x00000100000010db;\n" /* r0 = load_acquire((u64 *)(r1 + 0)) */ + "%[acq] = r0;" + : [acq]"=r"(acq) + : [args]"r"(args) + : "r0", "r1"); + if (acq != ptr) + return 0xbad; + *acq += 1; + if (args->null_ptr) + return 0xbad; + *args->nested.ptr += 1; + if (args->cookie != 0xc00c1e) + return 0xbad; + /* + * @faulting is a plain kernel pointer, so the member behind it is read + * with a probed load, which the JIT skips for a user address or which + * faults, and reads as NULL rather than as an arena pointer. + */ + if (args->faulting->ptr) + return 0xbad; + /* an arena pointer to kfuncs as well, the kfunc adds one */ + bpf_kfunc_arena_arg_test((u64 *)args->ptr); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member = { + .test_arena_member = (void *)test_arena_member_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr, its acquire load, nested.ptr and the kfunc each added one */ + if (*val != 45) + return 3; + + /* + * The callback saw (u32)(kaddr - kern_vm_start), the arena offset of + * val with the upper 32 bits clear, the same as an __arena argument. + */ + if (member_ptr_val != (u32)(u64)val) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c new file mode 100644 index 000000000000..0b4147894122 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reaches args->ptr on one path and args->cookie on the other, + * the branch on the unknown cookie value makes the verifier walk both. The + * rewrite is per insn, so the paths have to agree on what they load. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +__naked int mismatch_member(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = *(u64 *)(r6 + %[cookie_off]);" + "if r0 == 0 goto 1f;" + "r6 += %[cookie_off];" +"1: r0 = *(u64 *)(r6 + 0);" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena), + __imm_const(cookie_off, offsetof(struct bpf_testmod_arena_member_args, cookie)) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c new file mode 100644 index 000000000000..ff6d4db47cfd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reads args->ptr on one path and the memory argument on the + * other. The verifier lets a load insn serve a kernel struct pointer and a + * memory pointer, see save_aux_ptr_type(), but the rewrite of the arena + * member would corrupt the value the other path loads. A global function + * is the only place a struct_ops program of this type gets a memory pointer. + * The loaded value goes nowhere: an ALU insn on it would be an arena + * pointer on one path and a scalar on the other, which the verifier rejects + * on its own before the fixup pass gets to the load. + */ +__noinline int load_either(unsigned long long *ctx __arg_ctx, u64 *mem __arg_nonnull) +{ + struct bpf_testmod_arena_member_args *args = (void *)ctx[0]; + void *p = args; + + if (args->cookie == 0) + p = mem; + /* one load insn, reached with both pointers */ + asm volatile("r0 = *(u64 *)(%[p] + 0)" :: [p]"r"(p) : "r0"); + return 0; +} + +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +int BPF_PROG(mismatch_mem, struct bpf_testmod_arena_member_args *args) +{ + u64 local = 0; + + asm volatile("" :: "r"(&arena)); + return load_either(ctx, &local); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_mem, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c new file mode 100644 index 000000000000..cdd8ea740478 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the object: the member cannot be translated into anything. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct bpf_testmod_arena_member_args " + "requires a program with an associated arena") +int BPF_PROG(no_arena_member, struct bpf_testmod_arena_member_args *args) +{ + return args->ptr != NULL; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)no_arena_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c new file mode 100644 index 000000000000..d45666fd482c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * An offset into a union resolves to its first member, so a program reading + * raw there would get an arena pointer it did not ask for. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of bpf_testmod_arena_union is inside a union") +int BPF_PROG(union_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + return *args->u.ptr != 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)union_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c new file mode 100644 index 000000000000..ff0761980948 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The kernel owns the member, a program only reads it. A struct_ops program + * cannot store through its arguments at all, so the store is refused before + * the tag comes into play; the exchange test reaches the arena member check. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("only read is supported") +int BPF_PROG(write_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + args->ptr = NULL; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)write_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c new file mode 100644 index 000000000000..0d814166cabc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* An atomic that writes the member is neither a plain nor an acquire load. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct " + "bpf_testmod_arena_member_args can only be read with a plain or an acquire load") +__naked int xchg_member(void) +{ + asm volatile ( + "r1 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = 0;" + /* r0 = xchg_64((u64 *)(r1 + 0), r0): BPF_STX | BPF_ATOMIC | BPF_DW, imm BPF_XCHG */ + ".8byte 0x000000e1000001db;" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)xchg_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c new file mode 100644 index 000000000000..23fc2b927793 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include + +char _license[] SEC("license") = "GPL"; + +/* + * The module types as vmlinux.h renders them: the tag is an attribute on a + * pointer in the kernel address space, so the program has to cast the member + * to the arena address space before using it. That is what a program built + * against vmlinux.h sees for a kernel struct with an __arena member. + */ +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#define __nullable_tag __attribute__((btf_type_tag("nullable"))) + +struct bpf_testmod_arena_member_args { + __u64 __arena_tag *ptr; + __u64 __arena_tag __nullable_tag *null_ptr; + struct { + __u64 __arena_tag *ptr; + } nested; +} __attribute__((preserve_access_index)); + +struct bpf_testmod_ops3 { + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); +}; + +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; + +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_render_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = (u64 __arena *)args->ptr; + + arena_touch++; + *ptr += 1; + if (args->null_ptr) + return 0xbad; + *(u64 __arena *)args->nested.ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_render = { + .test_arena_member = (void *)test_arena_member_render_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr and nested.ptr each added one */ + if (*val != 43) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 37d5824810d1..6380f52f4dd2 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -506,6 +506,11 @@ static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair return 0; } +static int bpf_testmod_ops3__test_arena_member(struct bpf_testmod_arena_member_args *args) +{ + return 0; +} + static int bpf_testmod_ops3__test_nullable(struct bpf_testmod_nullable_args *args) { return 0; @@ -518,6 +523,7 @@ static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, .test_arena_stack = bpf_testmod_ops3__test_arena_stack, .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, + .test_arena_member = bpf_testmod_ops3__test_arena_member, .test_nullable = bpf_testmod_ops3__test_nullable, }; @@ -559,6 +565,23 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) return st_ops3->test_arena_multislot(p, ptr__arena); } +/* The callback gets the kernel address of @ptr__arena behind each __arena member. */ +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_member(u64 *ptr__arena) +{ + struct bpf_testmod_arena_member_args args = { + .ptr = ptr__arena, + .nested.ptr = ptr__arena, + .u.ptr = ptr__arena, + .cookie = 0xc00c1e, + /* in the first page, which is never mapped */ + .faulting = (void *)16, + }; + + if (!st_ops3 || !st_ops3->test_arena_member) + return -EOPNOTSUPP; + return st_ops3->test_arena_member(&args); +} + /* * The members of struct bpf_testmod_nullable_args are trusted the way the * verifier's BTF_TYPE_SAFE_TRUSTED() lists vmlinux members: through a struct @@ -1021,6 +1044,7 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_member) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_nullable) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 1a67564a6b8d..4ba9def74404 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -115,12 +115,25 @@ struct bpf_testmod_arena_pair { }; /* - * The qualifiers are BTF type tags for the verifier. The module gets them - * from bpf_compiler_types.h, user space and BPF programs need them empty. + * The qualifiers are BTF type tags for the verifier. The module gets __arena + * and __nullable from bpf_compiler_types.h. A BPF program gets __arena from + * bpf_arena_common.h, and the same definition here when it includes this + * header first, and needs __nullable and __percpu empty. User space needs all + * of them empty. */ #ifdef __KERNEL__ #include #endif +#if defined(__BPF__) && !defined(__arena) +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) +#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) +#else +#define __arena __attribute__((btf_type_tag("arena"))) +#endif +#endif +#ifndef __arena +#define __arena +#endif #ifndef __nullable #define __nullable #endif @@ -132,6 +145,31 @@ struct bpf_testmod_pcpu { int val; }; +struct bpf_testmod_arena_nested { + u64 __arena *ptr; +}; + +union bpf_testmod_arena_union { + u64 __arena *ptr; + u64 raw; +}; + +/* Arguments of test_arena_member(), the pointers hold kernel addresses of arena memory */ +struct bpf_testmod_arena_member_args { + /* the value the caller passed, never NULL */ + u64 __arena *ptr; + /* always NULL */ + u64 __arena __nullable *null_ptr; + /* the same value again, behind a nested struct */ + struct bpf_testmod_arena_nested nested; + /* the same value again, but a load from a union is rejected */ + union bpf_testmod_arena_union u; + /* a scalar, loaded by the same insn as @ptr in a verifier test */ + u64 cookie; + /* a bad pointer to a struct like this one, a load through it faults */ + struct bpf_testmod_arena_member_args *faulting; +}; + /* Arguments of test_nullable(), trusted through the __safe_trusted alias in the module */ struct bpf_testmod_nullable_args { /* never NULL */ @@ -157,6 +195,8 @@ struct bpf_testmod_ops3 { u64 g, u64 h, u64 *ptr); /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); + /* Used to test __arena pointer members. */ + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); /* Used to test __nullable pointer members. */ int (*test_nullable)(struct bpf_testmod_nullable_args *args); }; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 1e7b1f1e5148..3b810f1e438f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -224,6 +224,7 @@ int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_nullable(void) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, From bc4a327d5b871a850fffbc5e0e62e7b7acb9fa93 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:52 +0200 Subject: [PATCH 629/660] selftests/bpf: Move the no-arena tests out of objects with an arena map Two programs expect to fail for want of an arena while living next to an arena map: the kfunc argument case in arena_kfunc.c and the scalar access case in verifier_arena_scalar.c. Once libbpf binds the arena of an object to every program in it, they would load. Move them to an object without an arena map ahead of that change, where the kernel checks still apply, so that the tests pass at every step. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-11-memxor@gmail.com --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../testing/selftests/bpf/progs/arena_kfunc.c | 10 ----- .../selftests/bpf/progs/arena_noarena.c | 39 +++++++++++++++++++ .../bpf/progs/verifier_arena_scalar.c | 13 ------- 4 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/arena_noarena.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index c69c0f7e9cc8..6df440d82675 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -3,6 +3,7 @@ #include #include "arena_kfunc.skel.h" +#include "arena_noarena.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" #include "cb_refs.skel.h" @@ -193,6 +194,7 @@ static void run_tests_aux(const char *skel_name, } while (0) void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } +void test_arena_noarena(void) { RUN_TESTS(arena_noarena); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 96f7aacf4d9f..a049d6371d09 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -192,16 +192,6 @@ int arena_arg_unpopulated(void *ctx) return 0; } -SEC("syscall") -__arch_x86_64 -__arch_arm64 -__failure __msg("arena pointer requires a program with an associated arena") -int arena_arg_no_arena(void *ctx) -{ - bpf_kfunc_arena_arg_test((u64 *)1); - return 0; -} - SEC("syscall") __arch_x86_64 __arch_arm64 diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c new file mode 100644 index 000000000000..edbd40cd5be8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Programs without an arena. They cannot share an object with an arena map, + * libbpf binds the arena of an object to every program in it. + */ + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__flag(BPF_F_ARENA_SCALAR) +__arch_x86_64 +__arch_arm64 +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c index ecfc6873ffa9..6f0efd9202ca 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -991,19 +991,6 @@ __naked void no_flag(void) : __clobber_all); } -SEC("syscall") -__arena_scalar -__description("arena_scalar: no arena, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_arena(void) -{ - asm volatile (" \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" ::: __clobber_all); -} - SEC("syscall") __arena_scalar __description("arena_scalar: pointer that is NULL is an address in arena") From cd7a395838cbf76f5162fa48ff4a531be518fc43 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:53 +0200 Subject: [PATCH 630/660] libbpf: Bind the arena of an object to every program in it A program has an arena only when one of its insns refers to the arena map. The kernel can hand a program arena pointers all the same: as __arena struct_ops arguments, through __arena struct members and as kfunc return values. The verifier rejects those in a program without an arena, since it has nowhere to place them, and so a struct_ops callback that touches nothing but its arguments fails to load until it refers to the arena artificially, e.g. with asm volatile("" :: "r"(&arena)). Bind the arena of an object to every program loaded from it. The kernel binds the maps of a continuous fd_array (fd_array_cnt) like maps an insn refers to, so pass the arena in slot 0, which is free: a kfunc insn with offset 0 names vmlinux BTF rather than a slot. The module BTF slots that follow stay as they are. A feature probe keeps older kernels, which reject a non-zero fd_array_cnt, on the previous behavior. An object has at most one arena, libbpf refuses a second one, so there is no ambiguity about which arena a program gets. Programs that never touch the arena see it among their maps, and pay for the arena base setup in their JIT prologue until a later patch spares them that; the arena now lives as long as any program of the object. With the count, the kernel also resolves every module BTF slot of the array for every program of the object, so each of them holds the modules the object calls into, not only the programs that call them. Where the JIT is disabled or the loader lacks CAP_PERFMON, every program of such an object now fails to load, since an arena needs both, where before only the programs using the arena did. bpf_program__clone() binds the arena the same way, so a program loads through veristat as it does through bpf_object__load(). An arena exclusive to another program and the light skeleton loader are left alone. An arena exclusive to the program itself is bound only where the kernel checks an exclusive map against the digest of the final program. Earlier kernels compared a map bound through the fd_array against a digest they had not computed yet and refused the load for the very program the map is exclusive to, so a second probe loads a program with a map exclusive to it through fd_array_cnt; where that fails, the program keeps binding its arena through its insns, as before. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-12-memxor@gmail.com --- tools/lib/bpf/features.c | 82 +++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.c | 66 +++++++++++++++++++++++++- tools/lib/bpf/libbpf_internal.h | 4 ++ 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 9f589980beef..adb5d2e7fdfa 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -77,6 +77,82 @@ static int probe_kern_global_data(int token_fd) return probe_fd(ret); } +/* + * A continuous fd_array binds every map in it to the program, whether an + * insn refers to the map or not. Kernels without the attribute reject a + * non-zero fd_array_cnt. + */ +static int probe_kern_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) { + ret = -errno; + pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", + __func__, errstr(ret)); + return ret; + } + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + +/* + * The kernel checks an exclusive map bound through fd_array_cnt against the + * digest of the final program. Earlier kernels compared it as the map was + * added, against a digest they had not computed yet, and refused the load for + * the very program the map is exclusive to; kernels without exclusive maps + * refuse the map. + */ +static int probe_kern_excl_map_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + __u8 hash[SHA256_DIGEST_LENGTH]; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash), + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + libbpf_sha256(insns, sizeof(insns), hash); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_excl", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) + return 0; + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + static int probe_kern_btf(int token_fd) { static const char strs[] = "\0int"; @@ -742,6 +818,12 @@ static struct kern_feature_desc { [FEAT_PERCPU_DATA] = { "kernel supports percpu data", probe_kern_percpu_data, }, + [FEAT_FD_ARRAY_CNT] = { + "continuous fd_array (fd_array_cnt)", probe_kern_fd_array_cnt, + }, + [FEAT_EXCL_MAP_FD_ARRAY_CNT] = { + "exclusive map bound through fd_array_cnt", probe_kern_excl_map_fd_array_cnt, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fcedb78abe00..8fcb8cf39cc0 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -797,7 +797,8 @@ struct bpf_object { /* * Slots handed out so far, slot 0 included. A kfunc insn names a module * BTF slot through its offset, slot 0 stands for vmlinux BTF and stays - * empty, so the array goes to the kernel without a count. + * empty, so the array goes to the kernel without a count, unless + * bpf_object__bind_arena() puts the arena of the object there. */ size_t fd_array_used; @@ -8804,6 +8805,56 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); +/* + * Bind the arena of the object to the program about to be loaded. A program + * has an arena only when an insn refers to the map, but the kernel can hand + * it arena pointers all the same, as struct_ops arguments, through tagged + * struct members or as kfunc return values, and the verifier rejects those + * when the program has no arena to place them in. A continuous fd_array + * binds every map in it, the arena takes slot 0, which is free: a kfunc insn + * with offset 0 names vmlinux BTF rather than a slot. + */ +static int bpf_object__bind_arena(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_prog_load_opts *load_attr) +{ + struct bpf_map *map; + int err; + + if (obj->gen_loader || obj->arena_map_idx < 0 || !kernel_supports(obj, FEAT_FD_ARRAY_CNT)) + return 0; + map = &obj->maps[obj->arena_map_idx]; + /* + * Nothing to bind when the arena is not created, or no longer is after + * a failed load, when it is exclusive to another program, and nothing + * for an offloaded program, which the kernel does not let near an + * arena. + */ + if (!map->autocreate || map->fd < 0 || (map->excl_prog && map->excl_prog != prog) || + load_attr->prog_ifindex) + return 0; + /* + * An arena exclusive to the program itself needs a kernel that checks + * the map against the digest of the final program. Earlier kernels + * compared a map bound through the fd_array against a digest they had + * not computed yet and refused the load; there the program binds its + * arena through its insns, as before. + */ + if (map->excl_prog && !kernel_supports(obj, FEAT_EXCL_MAP_FD_ARRAY_CNT)) + return 0; + + if (!obj->fd_array_used) { + err = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 1); + if (err) + return err; + obj->fd_array_used = 1; + } + obj->fd_array[0] = map->fd; + load_attr->fd_array = obj->fd_array; + /* with slot 0 filled every slot handed out is valid, so the array can go with a count */ + load_attr->fd_array_cnt = obj->fd_array_used; + return 0; +} + static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, const char *license, __u32 kern_version, int *prog_fd) @@ -8867,6 +8918,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog if (obj->data_in_arena) load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; + err = bpf_object__bind_arena(obj, prog, &load_attr); + if (err) + return err; load_attr.token_fd = obj->token_fd; if (obj->token_fd) @@ -9966,8 +10020,10 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) static void bpf_object_post_load_cleanup(struct bpf_object *obj) { - /* clean up fd_array */ + /* clean up fd_array, and the slots handed out with it */ zfree(&obj->fd_array); + obj->fd_array_cap = 0; + obj->fd_array_used = 0; /* clean up BTF */ bpf_object_cleanup_btf(obj); @@ -10946,6 +11002,12 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); + /* the arena of the object goes with the program, as in bpf_object__load() */ + if (!OPTS_GET(opts, fd_array, NULL) && !OPTS_GET(opts, fd_array_cnt, 0)) { + err = bpf_object__bind_arena(obj, prog, &attr); + if (err) + return libbpf_err(err); + } attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4e45dfacb548..e90b5336bac0 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -404,6 +404,10 @@ enum kern_feature_id { FEAT_BPF_SYSCALL_COMMON_ATTRS, /* Kernel supports percpu data */ FEAT_PERCPU_DATA, + /* Kernel binds every map in a continuous fd_array (fd_array_cnt) to the program */ + FEAT_FD_ARRAY_CNT, + /* Kernel checks an exclusive map bound through fd_array_cnt against the final digest */ + FEAT_EXCL_MAP_FD_ARRAY_CNT, __FEAT_CNT, }; From 8911d776607317cd90a055111e8269da1a95b69b Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:54 +0200 Subject: [PATCH 631/660] selftests/bpf: Test arena binding by libbpf Attach a struct_ops callback with an __arena argument from an object whose callback and unrelated program never refer to the arena, drive it through the kernel, and check that an unrelated program of the object got the arena among its maps as well. Load the same object through bpf_object__prepare() and bpf_program__clone() too, the way veristat loads programs. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-13-memxor@gmail.com --- .../bpf/prog_tests/test_struct_ops_arena.c | 120 ++++++++++++++++++ .../bpf/progs/struct_ops_arena_autobind.c | 64 ++++++++++ 2 files changed, 184 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 1c04473fb42d..e7580b9f0fc2 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -13,6 +13,7 @@ #include "struct_ops_arena_member_fail__mismatch_mem.skel.h" #include "struct_ops_arena_member_fail__write.skel.h" #include "struct_ops_arena_member_fail__no_arena.skel.h" +#include "struct_ops_arena_autobind.skel.h" #if defined(__x86_64__) || defined(__aarch64__) /* @@ -145,6 +146,119 @@ static void arena_member(void) struct_ops_arena_member__destroy(skel); } +/* @prog_fd lists exactly one map, the one with @map_id. */ +static void assert_prog_map(int prog_fd, __u32 map_id, const char *name) +{ + __u32 map_ids[2] = {}, len = sizeof(struct bpf_prog_info); + struct bpf_prog_info info = { + .nr_map_ids = ARRAY_SIZE(map_ids), + .map_ids = ptr_to_u64(map_ids), + }; + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &len); + if (!ASSERT_OK(err, name)) + return; + if (!ASSERT_EQ(info.nr_map_ids, 1, name)) + return; + ASSERT_EQ(map_ids[0], map_id, name); +} + +/* + * libbpf binds the arena of an object to every program in it: a callback + * with an __arena argument loads without referring to the arena, and an + * unrelated program gets the arena as well. + */ +static void arena_autobind(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open_and_load")) + return; + + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.test_arena_cb), info.id, "test_arena_cb"); + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); + + attach_and_trigger(skel->maps.testmod_arena_autobind, skel->progs.trigger); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * The same binding through bpf_object__prepare() and bpf_program__clone(), the + * way veristat loads a program: the arena takes slot 0 of a continuous + * fd_array, next to the module BTF of the kfunc the trigger calls. + */ +static void arena_autobind_clone(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + struct bpf_program *prog; + int err, fd; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + bpf_object__for_each_program(prog, skel->obj) { + fd = bpf_program__clone(prog, NULL); + if (!ASSERT_GE(fd, 0, bpf_program__name(prog))) + continue; + assert_prog_map(fd, info.id, bpf_program__name(prog)); + close(fd); + } +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * An arena exclusive to a program is bound to that program all the same: the + * kernel checks an exclusive map against the digest of the final program, so + * the binding through the fd_array passes for the program the map is + * exclusive to. + */ +static void arena_autobind_excl(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + /* the other programs need the arena too, and cannot have it */ + bpf_map__set_autocreate(skel->maps.testmod_arena_autobind, false); + bpf_program__set_autoload(skel->progs.trigger, false); + err = bpf_map__set_exclusive_program(skel->maps.arena, skel->progs.unrelated); + if (!ASSERT_OK(err, "set_exclusive_program")) + goto out; + + err = struct_ops_arena_autobind__load(skel); + if (!ASSERT_OK(err, "load")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); +out: + struct_ops_arena_autobind__destroy(skel); +} + /* The same through the vmlinux.h rendering of the members, which takes casts. */ static void arena_member_render(void) { @@ -197,6 +311,12 @@ void serial_test_struct_ops_arena(void) /* so that the missing member tests show up as skipped */ test__skip(); } + if (test__start_subtest("arena_autobind")) + arena_autobind(); + if (test__start_subtest("arena_autobind_clone")) + arena_autobind_clone(); + if (test__start_subtest("arena_autobind_excl")) + arena_autobind_excl(); #else test__skip(); #endif diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c new file mode 100644 index 000000000000..495faa0f4a8b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_experimental.h" +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * Neither the callback nor the unrelated program refers to the arena. libbpf + * binds it to them all the same, so the callback can use the __arena argument + * the kernel hands it. + */ +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_autobind = { + .test_arena = (void *)test_arena_cb, +}; + +SEC("syscall") +int unrelated(void *ctx) +{ + return 0; +} + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (bpf_testmod_ops3_call_test_arena((u64 *)val)) + return 2; + if (*val != 42) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} From a8471eb0747f083ae5bde949e667d3d799d65cf7 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:55 +0200 Subject: [PATCH 632/660] bpf: Set up the arena base only in functions that use it A JIT sets up the arena base in the prologue of every function of a program that has an arena, a callee-saved register loaded with a 64-bit immediate and spilled around the function on x86. That is paid for by every program with an arena, whether it touches the arena or not, and now that libbpf binds the arena of an object to every program in it, by every program of an object that happens to contain an arena. Decide per function, on its final insns, right before it is JITed. bpf_prog_has_arena_insns() looks for an insn that works with the base: an access through an arena pointer, or through a number in a program loaded with BPF_F_ARENA_SCALAR, has been rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, which the JIT rebases, has a function model that says so. A load of an __arena struct member is rewritten with an immediate and needs none; dereferencing the result does. Looking at the final insns rather than recording the uses during verification keeps the answer in step with what the JIT sees. A constant NULL passed to a nullable __arena kfunc argument takes the early exit for NULL arguments in the verifier, and the JIT rebases the call all the same; an insn that is dead code has disappeared by then. bpf_prog_jit_compile() caches the answer in bpf_prog_aux for a single function program, jit_subprogs() for each subprogram once it has its kfunc table and arena. It implies that the program has an arena, as the scan says no without one. The JITs with arena support pick up the arena base and user address only when bpf_prog_uses_arena_insns() says so, and so generate the prologue and epilogue of a program bound to an arena it never touches as if it had none. The other users of the arena address of a program stay as they are on purpose. The struct_ops trampoline converts __arena arguments on entry and arena_member_fixup() converts loaded __arena members after the load, both with the base as an immediate. Both have to happen whether or not the program dereferences the result, it may compare, store or pass it on, so neither depends on the base register nor on the flag. The arena fault report reads the user address of the arena the faulting insn belongs to, and such an insn sets the flag anyway. check_map_prog_compatibility() checks the user address when the arena is bound, before any use is known. The rest, in arena.c and sched_ext, works with the map and not with a program. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-14-memxor@gmail.com --- arch/arm64/net/bpf_jit_comp.c | 6 ++-- arch/loongarch/net/bpf_jit.c | 6 ++-- arch/powerpc/net/bpf_jit_comp.c | 6 ++-- arch/riscv/net/bpf_jit_core.c | 6 ++-- arch/s390/net/bpf_jit_comp.c | 7 ++-- arch/x86/net/bpf_jit_comp.c | 14 ++++---- include/linux/bpf.h | 2 ++ include/linux/filter.h | 13 ++++++++ kernel/bpf/core.c | 57 +++++++++++++++++++++++++++++++++ kernel/bpf/fixups.c | 1 + 10 files changed, 101 insertions(+), 17 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 506c4201fec8..1d5f4cf88d33 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2284,8 +2284,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr if (ctx.offset == NULL) goto out_off; - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + } out_cnt = bpf_out_stack_arg_cnt(env, prog); if (out_cnt) { diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index e78eb582c300..9393e4b0db8e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2245,8 +2245,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); if (ctx.offset == NULL) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7a612688e7a2..0d6ea6df0106 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -241,8 +241,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); - cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); - cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + if (bpf_prog_uses_arena_insns(fp)) { + cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); + cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + } cgctx.is_subprog = bpf_is_subprog(fp); cgctx.exception_boundary = fp->aux->exception_boundary; cgctx.exception_cb = fp->aux->exception_cb; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 5265dd5bd39e..f2497ec7d378 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -73,8 +73,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto skip_init_ctx; } - ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); /* First 3 stack args in regs, rest on stack */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 14c905697352..8eaaac7028a1 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2304,10 +2304,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, else jit->frame_off = 0; - kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); - if (kern_arena) + if (bpf_prog_uses_arena_insns(fp)) { + kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); jit->kern_arena = _EMIT_CONST_U64(kern_arena); - jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + } bpf_jit_prologue(jit, fp); if (bpf_set_addr(jit, 0) < 0) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d3843045dc71..72a2a0a89e03 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -888,7 +888,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -954,7 +954,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -2443,7 +2443,7 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; - if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + if (WARN_ON_ONCE(!bpf_prog_uses_arena_insns(bpf_prog))) return -EINVAL; /* mov eN, eN: truncate and clear the upper 32 bits */ @@ -2522,7 +2522,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; void __percpu *priv_frame_ptr = NULL; u16 out_stack_arg_cnt, outgoing_rsp; - u64 arena_vm_start, user_vm_start; + u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; int ilen, proglen = 0; @@ -2562,8 +2562,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * [below] callee-saved regs * [below] outgoing args 7+ (= rsp) */ - arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); - user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + if (bpf_prog_uses_arena_insns(bpf_prog)) { + arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + } detect_reg_usage(insn, insn_cnt, callee_regs_used); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index dae33b1a7427..7750df973494 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1898,6 +1898,8 @@ struct bpf_prog_aux { bool has_prologue; bool might_sleep; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; diff --git a/include/linux/filter.h b/include/linux/filter.h index 9339c6131f8f..6f4c9bd024a0 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1110,6 +1110,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e32db9777c2..fc22d66d194a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2637,11 +2637,68 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } +/* + * Whether a function has an insn that works with the arena base, which a JIT + * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on + * the final insns of the function: an access through an arena pointer, or + * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address + * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, + * which the JIT rebases, has a function model that says so. A load of an + * __arena struct member is rewritten with an immediate and needs none; + * dereferencing the result does. + */ +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + const struct btf_func_model *m; + int i, j; + + if (!prog->aux->arena) + return false; + + for (i = 0; i < prog->len; i++, insn++) { + switch (BPF_CLASS(insn->code)) { + case BPF_LDX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) + return true; + break; + case BPF_ST: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) + return true; + break; + case BPF_STX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) + return true; + break; + case BPF_ALU64: + if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && + insn->off == BPF_ADDR_SPACE_CAST) + return true; + break; + case BPF_JMP: + if (!bpf_pseudo_kfunc_call(insn)) + break; + m = bpf_jit_find_kfunc_model(prog, insn); + if (!m) + break; + for (j = 0; j < m->nr_args; j++) + if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) + return true; + break; + } + } + return false; +} + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; + prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 34b53ebb5841..409b9104c723 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1448,6 +1448,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; + func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; From 7418d9291c57ea563b5fa706dea0883effe5d295 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:56 +0200 Subject: [PATCH 633/660] libbpf: Render the arena type tag with its address space A kernel struct member tagged __arena reaches a program built against vmlinux.h as a pointer in the kernel address space with the tag as an attribute, struct scx_cmask __attribute__((btf_type_tag("arena"))) *cmask, so the program casts it to the arena address space before dereferencing it, although the verifier already hands it an arena pointer. The tag denotes a pointer into the BPF arena, address_space(1) to the compiler, so render it as such: __attribute__((address_space(1))) followed by the tag attribute as before, which keeps the dump faithful to the BTF. A program then reads the member as an arena pointer and hands it to anything declared with its own __arena without a cast. Nothing else changes for consumers of the C dump. Skeletons strip the modifiers of variable types and never carried the tag. Clang without casts between address spaces still accepts the declaration and only a program touching such a member would need a cast there, as it did for every arena pointer before. GCC has no address spaces for BPF and warns, under -Wattributes, that it ignores the attribute, which the selftests build already silences. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-15-memxor@gmail.com --- tools/lib/bpf/btf_dump.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 996e61b5f11f..7dedd3975c39 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1595,6 +1595,9 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, name = btf_name_of(d, t->name_off); if (btf_kflag(t)) btf_dump_printf(d, " __attribute__((%s))", name); + else if (strcmp(name, "arena") == 0) + /* the tag denotes a pointer into the BPF arena address space */ + btf_dump_printf(d, " __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\")))"); else btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name); break; From 31ef8ab6dfc7152e4b64bcae19a3e07b10dd8fea Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:57 +0200 Subject: [PATCH 634/660] selftests/bpf: Test the address space rendering of the arena type tag Add an arena-tagged pointer member to the type tag dump test and expect the address space attribute in front of the tag. The struct_ops test that mirrors the previous rendering now describes it as the output of an older libbpf. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-16-memxor@gmail.com --- .../testing/selftests/bpf/prog_tests/btf_dump.c | 16 +++++++++++++--- .../bpf/progs/struct_ops_arena_member_render.c | 8 ++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 17fa44cb9c91..35f214e254da 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -604,8 +604,10 @@ static void test_btf_dump_type_tags(void) * struct s { * void __attribute__((btf_type_tag(\"void_tag\"))) *p1; * void __attribute__((void_attr)) *p2; + * void __arena *p3; * }; * + * The arena tag is rendered with the address space it denotes. */ id = btf__add_type_tag(btf, "void_tag", 0); @@ -618,17 +620,25 @@ static void test_btf_dump_type_tags(void) id = btf__add_ptr(btf, id); ASSERT_EQ(id, 4, "void_ptr_id2"); - id = btf__add_struct(btf, "s", 8); - ASSERT_EQ(id, 5, "struct_id"); + id = btf__add_type_tag(btf, "arena", 0); + ASSERT_EQ(id, 5, "arena_tag_id"); + id = btf__add_ptr(btf, id); + ASSERT_EQ(id, 6, "void_ptr_id3"); + + id = btf__add_struct(btf, "s", 24); + ASSERT_EQ(id, 7, "struct_id"); err = btf__add_field(btf, "p1", 2, 0, 0); ASSERT_OK(err, "field_ok1"); - err = btf__add_field(btf, "p2", 4, 0, 0); + err = btf__add_field(btf, "p2", 4, 64, 0); ASSERT_OK(err, "field_ok2"); + err = btf__add_field(btf, "p3", 6, 128, 0); + ASSERT_OK(err, "field_ok3"); test_ctx__dump_and_compare(&t, "struct s {\n" " void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n" " void __attribute__((void_attr)) *p2;\n" +" void __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\"))) *p3;\n" "};\n\n", "dump_and_compare"); test_ctx__free(&t); diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c index 23fc2b927793..09b4c740bdca 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c @@ -11,10 +11,10 @@ char _license[] SEC("license") = "GPL"; /* - * The module types as vmlinux.h renders them: the tag is an attribute on a - * pointer in the kernel address space, so the program has to cast the member - * to the arena address space before using it. That is what a program built - * against vmlinux.h sees for a kernel struct with an __arena member. + * The module types as a vmlinux.h generated by a libbpf that does not render + * the arena tag with its address space shows them: an attribute on a pointer + * in the kernel address space, so the program has to cast the member to the + * arena address space before using it. */ #define __arena_tag __attribute__((btf_type_tag("arena"))) #define __nullable_tag __attribute__((btf_type_tag("nullable"))) From fb854e2e54a7b0d19f6d93342ae0a64954411869 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 15:21:58 +0200 Subject: [PATCH 635/660] selftests/bpf: Test an arena kfunc argument passed from a subprogram Call a kfunc with an __arena argument from a global subprogram that does nothing else with the arena, the address arrives as a number and no cast is involved. The subprogram needs the arena base set up by its own prologue for the JIT to rebase the argument, which the BPF CI review found was not recorded for it. Pass a NULL constant to a nullable __arena kfunc argument as well, from a program that touches the arena nowhere else. The verifier takes its early exit for a NULL argument, but the JIT rebases the call like any other, behind the NULL test the function model asks for, so the arena base has to be set up for that call alone. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019132203.3034698-17-memxor@gmail.com --- .../testing/selftests/bpf/progs/arena_kfunc.c | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index a049d6371d09..5d1cb1a7e8ee 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -192,6 +192,57 @@ int arena_arg_unpopulated(void *ctx) return 0; } +/* + * A subprogram whose only arena use is the argument of a kfunc needs the + * arena base too, the JIT rebases the argument on it. The address arrives + * as a number, so no cast marks the subprogram either. + */ +__noinline int arena_arg_from_subprog(unsigned long addr) +{ + return bpf_kfunc_arena_arg_test((u64 *)addr) != 41; +} + +/* + * A NULL constant for the __arena argument takes an early exit in the + * verifier; the call needs the arena base all the same, the JIT rebases + * the argument with a NULL test. + */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_null(void *ctx) +{ + return bpf_kfunc_arena_cap_nullable_test(NULL) != 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_subprog(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (arena_arg_from_subprog((unsigned long)val)) + ret = 2; + else + ret = *val != 42 ? 3 : 0; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); + return ret; +#else + return 0; +#endif +} + SEC("syscall") __arch_x86_64 __arch_arm64 From 63be110a96aea4d0b5d5fe34cbd3ecf0819e14a3 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Mon, 19 Oct 2026 17:10:04 -0400 Subject: [PATCH 636/660] bpf: Drop unused entries from special_kfunc_list again Commit 6f033615ef8f ("bpf: Trim special_kfunc_list in verifier") dropped bpf_arena_reserve_pages, bpf_stream_vprintk and bpf_stream_print_stack from special_kfunc_list once nothing used them anymore, but commit ab7fff217fd1 ("bpf: Check helper and kfunc arguments in one path") brought all three back without adding any users. Besides being dead, the bpf_arena_reserve_pages entry can't be resolved on 32-bit and !MMU kernels, which don't build arena.o, and since commit 201a454df948 ("resolve_btfids: Fix the --fatal_warnings exit status") the resulting resolve_btfids warning fails the build with CONFIG_WERROR=y: WARN: resolve_btfids: unresolved symbol bpf_arena_reserve_pages Drop the three entries again. Fixes: ab7fff217fd1 ("bpf: Check helper and kfunc arguments in one path") Signed-off-by: Sasha Levin Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019211006.2574350-2-sashal@kernel.org --- kernel/bpf/verifier.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 015dc1bccebe..d49b7ea2c002 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13612,10 +13612,7 @@ enum special_kfunc_type { KF_bpf_call_rcu_tasks_trace, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, - KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, - KF_bpf_stream_vprintk, - KF_bpf_stream_print_stack, }; BTF_ID_LIST(special_kfunc_list) @@ -13707,14 +13704,11 @@ BTF_ID(func, bpf_call_rcu) BTF_ID(func, bpf_call_rcu_tasks_trace) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) -BTF_ID(func, bpf_arena_reserve_pages) #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif -BTF_ID(func, bpf_stream_vprintk) -BTF_ID(func, bpf_stream_print_stack) static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) { From ced6a0993e2f89158451a1b8651169ac20725aee Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Mon, 19 Oct 2026 17:10:05 -0400 Subject: [PATCH 637/660] bpf: Don't reference arena kfuncs when arena isn't built kernel/bpf/Makefile builds arena.o only for CONFIG_MMU && CONFIG_64BIT, but special_kfunc_list references bpf_arena_alloc_pages() and bpf_arena_free_pages() unconditionally, so on 32-bit and !MMU kernels resolve_btfids can't resolve them: WARN: resolve_btfids: unresolved symbol bpf_arena_free_pages WARN: resolve_btfids: unresolved symbol bpf_arena_alloc_pages Since commit 201a454df948 ("resolve_btfids: Fix the --fatal_warnings exit status") these warnings fail the build with CONFIG_WERROR=y, which breaks i386 builds with CONFIG_DEBUG_INFO_BTF=y: Failed to generate BTF for vmlinux Try to disable CONFIG_DEBUG_INFO_BTF Only reference them under the condition arena.o is built with, and use BTF_ID_UNUSED otherwise to keep the list in sync with enum special_kfunc_type. An unresolved ID is zero, like BTF_ID_UNUSED, so the contents of the list don't change. Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe") Signed-off-by: Sasha Levin Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019211006.2574350-3-sashal@kernel.org --- kernel/bpf/verifier.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d49b7ea2c002..264da570a0a5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13702,8 +13702,13 @@ BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_call_rcu) BTF_ID(func, bpf_call_rcu_tasks_trace) +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) +#else +BTF_ID_UNUSED +BTF_ID_UNUSED +#endif #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else From a81bd9262cde4b002b95e652453ce5b815c79a62 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Mon, 19 Oct 2026 17:10:06 -0400 Subject: [PATCH 638/660] bpf: Fix ksock build with BPF_LSM=y and SECURITY_NETWORK=n bpf_ksock.c references bpf_lsm_socket_sendmsg() whenever CONFIG_BPF_LSM=y, but the socket LSM hooks, and with them the BPF LSM socket_sendmsg hook, only exist with CONFIG_SECURITY_NETWORK=y. Without it resolve_btfids warns: WARN: resolve_btfids: unresolved symbol bpf_lsm_socket_sendmsg and since commit 201a454df948 ("resolve_btfids: Fix the --fatal_warnings exit status") that fails the build with CONFIG_WERROR=y. Only reference the hook with CONFIG_SECURITY_NETWORK=y, and use BTF_ID_UNUSED otherwise. The ID stays zero, as it was when unresolved, so bpf_ksock_kfunc_filter() keeps allowing LSM programs: none can be attached to socket_sendmsg without CONFIG_SECURITY_NETWORK. Fixes: 7ae4eb14c5f9 ("bpf: Add ksock kfuncs") Signed-off-by: Sasha Levin Signed-off-by: Alexei Starovoitov Acked-by: Mahe Tardy Link: https://patch.msgid.link/20261019211006.2574350-4-sashal@kernel.org --- net/core/bpf_ksock.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/net/core/bpf_ksock.c b/net/core/bpf_ksock.c index 7bf796193ab6..48a2e2327ace 100644 --- a/net/core/bpf_ksock.c +++ b/net/core/bpf_ksock.c @@ -258,7 +258,12 @@ BTF_ID_FLAGS(func, bpf_ksock_send, KF_SLEEPABLE) BTF_KFUNCS_END(ksock_kfunc_btf_ids) #ifdef CONFIG_BPF_LSM -BTF_ID_LIST_SINGLE(bpf_lsm_socket_sendmsg_id, func, bpf_lsm_socket_sendmsg) +BTF_ID_LIST(bpf_lsm_socket_sendmsg_id) +#ifdef CONFIG_SECURITY_NETWORK +BTF_ID(func, bpf_lsm_socket_sendmsg) +#else +BTF_ID_UNUSED +#endif #endif static int bpf_ksock_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) From 19a17b5b2b01f6b5ba8f8d493629c759f90906d5 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 16:48:49 +0200 Subject: [PATCH 639/660] bpf: Refuse CO-RE relocations for a signed program The signature of a program covers its insns as submitted, along with the frozen metadata maps of a signed loader, and bpf_prog_verify_signature() checks it before the verifier does anything else with the program. CO-RE relocations are the one input besides the insns that changes them: the loader hands them in with the program, and bpf_check_core_relo() applies them after the signature check, rewriting the immediate or the offset of every insn a record names with a value the loader chooses, a field offset, a type size or id or an enum value of the kernel BTF, or a type id of the BTF the loader supplies itself. Whoever holds a signed program can so change its constants once the signature has passed, the offsets a signed loader reads its metadata map with for instance, and run a program the signer never saw. The exclusive metadata map of a signed loader does not make up for it: the hash that binds the map to a program is chosen by whoever creates the map, so it can name the relocated program as well. Refuse a signed load that carries relocations, before the signature is examined, like the other checks on the shape of a signed load. A signed loader has none to lose: the programs it embeds are relocated by the loader itself through bpf_core_apply(), and nothing relocates the loader. The BTF of the program itself can stay: without relocations it steers only the verification, through func_info, line_info and the annotations of global function arguments, and rewrites nothing. Fixes: 349271568303 ("bpf: Implement signature verification for BPF programs") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Daniel Borkmann Link: https://patch.msgid.link/20261019144851.4052056-2-memxor@gmail.com --- kernel/bpf/verifier.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 264da570a0a5..01434e1d23b5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23501,6 +23501,15 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, u64 data_sz; int err = 0; + /* + * The signature covers the insns as submitted. CO-RE relocations would + * have the kernel rewrite them afterwards, with values the loader picks + * through the BTF it supplies, see bpf_check_core_relo(). + */ + if (attr->core_relo_cnt) { + verbose(env, "signed program cannot carry CO-RE relocations\n"); + return -EACCES; + } if (!attr->signature_size || attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; From 25a2e54999b0b93e8ecebb5863e3ecbcafb0ed18 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 19 Oct 2026 16:48:50 +0200 Subject: [PATCH 640/660] selftests/bpf: Test that a signed program with CO-RE relocations is refused Load a two insn program with a junk signature and one CO-RE relocation record, and check that the kernel refuses it with -EACCES and the message naming the relocations, before it examines the signature. Without the check the same load gets as far as the signature and fails on it with -EBADMSG. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Daniel Borkmann Link: https://patch.msgid.link/20261019144851.4052056-3-memxor@gmail.com --- .../selftests/bpf/prog_tests/signed_loader.c | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index a0f93756e717..e43dba8fed8e 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -379,6 +379,51 @@ static void signature_enforced(void) gen_loader_fixture_fini(&f); } +/* + * CO-RE relocations would have the kernel rewrite the insns after the + * signature was checked over them, with values the loader picks through the + * BTF it supplies. A signed load that carries any is rejected before the + * signature is examined, so junk serves as one. + */ +static void signed_core_relo_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_core_relo relo = { + .insn_off = 0, + .type_id = 1, + .kind = BPF_CORE_TYPE_ID_LOCAL, + }; + char log_buf[1024] = {}; + union bpf_attr attr; + int fd; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.core_relos = ptr_to_u64(&relo); + attr.core_relo_cnt = 1; + attr.core_relo_rec_size = sizeof(relo); + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); + fd = fd < 0 ? -errno : fd; + ASSERT_EQ(fd, -EACCES, "signed load with CO-RE relocations rejected"); + ASSERT_HAS_SUBSTR(log_buf, "signed program cannot carry CO-RE relocations", "log"); + if (fd >= 0) + close(fd); +} + static void signed_nonexcl_fd_array_rejected(void) { static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; @@ -2187,6 +2232,7 @@ static const struct { { "signed_unfrozen_fd_array_rejected", signed_unfrozen_fd_array_rejected, BOOT_SEALED }, { "signed_nonarray_fd_array_rejected", signed_nonarray_fd_array_rejected, BOOT_SEALED }, { "signed_btf_fd_array_rejected", signed_btf_fd_array_rejected, BOOT_ANY }, + { "signed_core_relo_rejected", signed_core_relo_rejected, BOOT_ANY }, { "signed_module_kfunc_rejected", signed_module_kfunc_rejected, BOOT_SEALED }, { "signature_failure_logs", signature_failure_logs, BOOT_SEALED }, { "signature_too_large", signature_too_large, BOOT_ANY }, From 05bee0638bacf4e9298bbf0bfef749b6a8ff4478 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:06 +0000 Subject: [PATCH 641/660] bpf: Revalidate faulted arena pages before installing user mappings A user fault on an arena pins the page that the kernel arena maps, drops the arena spinlock and returns the page to the core fault code, which then locks the folio and installs the user PTE. A concurrent free can clear the kernel mapping in that window. The page reference prevents a use-after-free, but the fault can still install a user PTE for a page the arena has already released. User space and BPF programs then see different memory at the same arena offset. Close the window from the fault side. Lock the page, recheck that the kernel mapping still points to it, and return it with VM_FAULT_LOCKED so that it stays locked until the core fault code has installed the user PTE. If a free removed the page, drop it and return VM_FAULT_NOPAGE so that the fault starts over. Return VM_FAULT_HWPOISON for a poisoned page. This change alone does not close the race, because the free path does not yet wait for the page lock. The following change makes frees wait for in-flight faults before zapping user mappings. Fixes: 317460317a02 ("bpf: Introduce bpf_arena.") Suggested-by: Alexei Starovoitov Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-2-emil@etsalapatis.com --- kernel/bpf/arena.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 0ff707707da0..5e0e0837980b 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -584,8 +584,22 @@ out: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); if (new_page) free_pages_nolock(new_page, 0); - vmf->page = page; - return 0; + + /* + * A free clears the kernel pte and then takes the page lock before it + * zaps user vma-s. Either it waits for this fault and zaps its pte, or + * the page is gone from here. + */ + lock_page(page); + if (vmalloc_to_page((void *)kaddr) == page && !PageHWPoison(page)) { + vmf->page = page; + return VM_FAULT_LOCKED; + } + /* The page was freed meanwhile. Let the fault start over. */ + fault_ret = PageHWPoison(page) ? VM_FAULT_HWPOISON : VM_FAULT_NOPAGE; + unlock_page(page); + put_page(page); + return fault_ret; out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); From 7244a2e58d2167292972f9380abb3a4d17a343e3 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:07 +0000 Subject: [PATCH 642/660] bpf: Fix arena race between page free and alloc leading to incoherency Existing arena kfunc code has an underlying race condition that can lead to writes being lost from the BPF program's point of view: a) A memory range gets freed by operation (1), and its range is added back to the arena range tree. b) A concurrent allocation (2) reallocates the range, and does writes to it. Writes from that CPU may follow the stale TLB entries into the pages that are about to be freed. c) (1) invalidates the TLB. The old pages, and any writes done to them, are now inaccessible. zap_pages() similarly removes the mappings for userspace threads. This can be triggered by particularly demanding BPF arena data structures that constantly allocate and deallocate memory, like hash table allocations. Solve this ABA problem by keeping the range allocated until TLB invalidation and unmapping are complete. Under the arena spinlock, claim any free pages in the range and clear its kernel PTEs. Then drop the spinlock, lock and unlock each cleared page to wait for faults that are still installing a user PTE, flush the kernel TLB and zap the user page tables. Finally, retake the spinlock to release the range. Frees serialize on zap_mutex, so an overlapping free cannot release the range early. Claiming the range only allocates a range tree node when the whole range is free, and then there is nothing to free. User faults on a range that is allocated without a page now raise SIGBUS instead of populating it. That covers a range reserved with bpf_arena_reserve_pages() and a range whose kernel PTEs a free has already cleared. Touching a page while a BPF program frees it is a user space bug, and in a BPF_F_SEGV_ON_FAULT arena it already raises SIGSEGV. Sleepable frees run synchronously and are deferred only when the arena spinlock cannot be taken. The deferred free worker retries such a free in place instead of requeueing itself, so the flush_work() in arena_map_free() waits for all of its work. Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe") Reported-by: Mykola Lysenko Suggested-by: Alexei Starovoitov Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-3-emil@etsalapatis.com --- kernel/bpf/arena.c | 159 +++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 86 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 5e0e0837980b..868589f07330 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -543,6 +543,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) goto out_err_locked; } + if (is_range_tree_set(&arena->rt, vmf->pgoff, 1)) { + /* Allocated without a page: reserved or being freed */ + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); if (!new_page) { @@ -868,7 +874,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) * otherwise call B's mark would make call A skip a vml that A has * not yet zapped for A's uaddr range. */ - mutex_lock(&arena->zap_mutex); + lockdep_assert_held(&arena->zap_mutex); mutex_lock(&arena->lock); my_gen = ++arena->zap_gen; for (;;) { @@ -902,49 +908,41 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) mutex_lock(&arena->lock); } mutex_unlock(&arena->lock); - mutex_unlock(&arena->zap_mutex); } -static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +/* + * The range stays allocated in the range tree until the pages are unmapped + * from vmalloc area and from all user space vma-s, so that nobody can + * populate it while stale TLB entries or user ptes are still around. + */ +static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) { - struct mem_cgroup *new_memcg, *old_memcg; - u64 full_uaddr, uaddr_end; - long kaddr, pgoff; - struct page *page; - struct llist_head free_pages; - struct llist_node *pos, *t; - struct arena_free_span *s; + u64 full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + long kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + long pgoff = compute_pgoff(arena, uaddr); struct clear_range_data cdata; + struct llist_node *pos, *t; + LLIST_HEAD(free_pages); unsigned long flags; - int ret = 0; + struct page *page; + int ret; - /* only aligned lower 32-bit are relevant */ - uaddr = (u32)uaddr; - uaddr &= PAGE_MASK; - kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) - return; - uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); - if (full_uaddr >= uaddr_end) - return; - - page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; - pgoff = compute_pgoff(arena, uaddr); - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - - if (!sleepable) - goto defer; + /* An overlapping free must not release the range before this one is done */ + guard(mutex)(&arena->zap_mutex); ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags); - - /* Can't proceed without holding the spinlock so defer the free */ if (ret) - goto defer; + return ret; - range_tree_set(&arena->rt, pgoff, page_cnt); + /* + * Take the free pages of the range too. It allocates, and can fail, + * only when the whole range is free. Then there is nothing to do. + */ + if (range_tree_clear(&arena->rt, pgoff, page_cnt)) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; + } - init_llist_head(&free_pages); cdata.arena = arena; cdata.free_pages = &free_pages; /* clear ptes and collect struct pages */ @@ -954,6 +952,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + /* wait for arena_vm_fault() that found the page before its pte was cleared */ + llist_for_each_entry(page, free_pages.first, pcp_llist) { + lock_page(page); + unlock_page(page); + } + /* ensure no stale TLB entries */ flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); @@ -972,8 +976,39 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, zap_pages(arena, full_uaddr, 1); __free_page(page); } - bpf_map_memcg_exit(old_memcg, new_memcg); + /* The callers hold no other rqspinlock, so this one can only time out */ + while (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + cond_resched(); + range_tree_set(&arena->rt, pgoff, page_cnt); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +{ + struct mem_cgroup *new_memcg, *old_memcg; + u64 full_uaddr, uaddr_end; + struct arena_free_span *s; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + /* Can't proceed without holding the spinlock so defer the free */ + if (!sleepable || arena_free_range(arena, uaddr, page_cnt)) + goto defer; + + bpf_map_memcg_exit(old_memcg, new_memcg); return; defer: @@ -1035,66 +1070,18 @@ static void arena_free_worker(struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); struct mem_cgroup *new_memcg, *old_memcg; - struct llist_node *list, *pos, *t; + struct llist_node *pos, *t; struct arena_free_span *s; - u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; - struct clear_range_data cdata; - struct page *page; - unsigned long full_uaddr; - long kaddr, page_cnt, pgoff; - unsigned long flags; - - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { - schedule_work(work); - return; - } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - init_llist_head(&free_pages); - cdata.arena = arena; - cdata.free_pages = &free_pages; - arena_vm_start = bpf_arena_get_kern_vm_start(arena); - user_vm_start = bpf_arena_get_user_vm_start(arena); - - list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, llist_del_all(&arena->free_spans)) { s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - kaddr = arena_vm_start + s->uaddr; - pgoff = compute_pgoff(arena, s->uaddr); - - /* clear ptes and collect pages in free_pages llist */ - apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); - } - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - - /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { - s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - full_uaddr = clear_lo32(user_vm_start) + s->uaddr; - kaddr = arena_vm_start + s->uaddr; - - /* ensure no stale TLB entries */ - flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); - - /* remove pages from user vmas */ - zap_pages(arena, full_uaddr, page_cnt); - + while (arena_free_range(arena, s->uaddr, s->page_cnt)) + cond_resched(); kfree_nolock(s); } - /* free all pages collected by apply_to_existing_page_range() in the first loop */ - llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { - page = llist_entry(pos, struct page, pcp_llist); - __free_page(page); - } - bpf_map_memcg_exit(old_memcg, new_memcg); } From 6c372031078cfa4c1ee1a789d29108af794ec1ab Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:08 +0000 Subject: [PATCH 643/660] selftests/bpf: Add arena allocation race tests Add selftests for races between arena frees, reallocation and user faults: - realloc_during_free, realloc_during_deferred_free: a synchronous or deferred free keeps its range allocated until it completes. - fault_free_realloc: user and BPF views of a page stay coherent while it is repeatedly faulted, freed and reallocated. Pause frees in zap_vma_range(), after the range's kernel mappings are gone. Unlike flush_tlb_kernel_range(), which is inline on arm64, it is generic MM code that fentry can attach to. Map the page into user space first, so that the free has to zap it. Print why a subtest is skipped when a tracing target is missing. The reader in fault_free_realloc skips reads that raise SIGBUS, which a fault on a page that is being freed does. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-4-emil@etsalapatis.com --- .../selftests/bpf/prog_tests/arena_race.c | 320 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_race.c | 166 +++++++++ 2 files changed, 486 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_race.c create mode 100644 tools/testing/selftests/bpf/progs/arena_race.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_race.c b/tools/testing/selftests/bpf/prog_tests/arena_race.c new file mode 100644 index 000000000000..f9d0e9c3a006 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_race.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "arena_race.skel.h" + +struct free_thread_ctx { + struct arena_race *skel; + int err; + __u32 retval; +}; + +struct fault_thread_ctx { + __u64 *addr; + int stop; +}; + +static int run_prog(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(err, name) && ASSERT_OK(opts.retval, name) ? 0 : -1; +} + +/* Trigger the sleepable free page path. */ +static void *run_free_thread(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct free_thread_ctx *ctx = arg; + + ctx->skel->bss->target_tid = sys_gettid(); + ctx->err = bpf_prog_test_run_opts( + bpf_program__fd(ctx->skel->progs.free_page), &opts); + ctx->retval = opts.retval; + return NULL; +} + +static __thread sigjmp_buf reader_env; +static __thread bool reader_armed; + +/* Only the reader expects SIGBUS; anywhere else it keeps its default action. */ +static void reader_sigbus(int sig) +{ + if (!reader_armed) { + signal(sig, SIG_DFL); + raise(sig); + return; + } + siglongjmp(reader_env, 1); +} + +/* Continuously fault in the address. */ +static void *fault_reader_thread(void *arg) +{ + struct fault_thread_ctx *ctx = arg; + __u64 *addr; + + /* Touching a page while BPF frees it raises SIGBUS; skip that read. */ + sigsetjmp(reader_env, 1); + reader_armed = true; + while (!READ_ONCE(ctx->stop)) { + addr = __atomic_load_n(&ctx->addr, __ATOMIC_ACQUIRE); + (void)READ_ONCE(*addr); + } + reader_armed = false; + return NULL; +} + +static int wait_for(int *p) +{ + __u64 deadline = get_time_ns() + 5ULL * 1000 * 1000 * 1000; + + while (!READ_ONCE(*p)) { + if (get_time_ns() > deadline) + return -ETIMEDOUT; + } + return 0; +} + +/* Skip the subtest, saying why, if @func cannot be traced with fentry. */ +static bool require_fentry_target(const char *test, const char *func) +{ + if (libbpf_find_vmlinux_btf_id(func, BPF_TRACE_FENTRY) > 0) + return true; + printf("%s:SKIP: %s is not an fentry target\n", test, func); + test__skip(); + return false; +} + +static struct arena_race *setup_arena(__u64 **addr, bool trace_zap) +{ + struct arena_race *skel; + size_t arena_sz; + char *base; + int err; + + skel = arena_race__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + err = bpf_program__set_autoload(skel->progs.trace_zap, trace_zap); + if (!ASSERT_OK(err, "trace_zap_autoload")) + goto err_out; + + err = arena_race__load(skel); + if (!ASSERT_OK(err, "load")) + goto err_out; + err = arena_race__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err_out; + + if (run_prog(skel->progs.alloc_old, "alloc_old")) + goto err_out; + if (skel->bss->skip) { + printf("%s:SKIP: compiler doesn't support arena_cast\n", __func__); + test__skip(); + goto err_out; + } + + base = bpf_map__initial_value(skel->maps.arena, &arena_sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto err_out; + *addr = (__u64 *)(base + getpagesize()); + if (!ASSERT_EQ((unsigned long)skel->bss->ptr, (unsigned long)*addr, + "arena_ptr")) + goto err_out; + return skel; + +err_out: + arena_race__destroy(skel); + return NULL; +} + +static void test_realloc_during_free(bool deferred) +{ + struct free_thread_ctx ctx = {}; + struct arena_race *skel; + pthread_t thread; + __u64 *addr; + bool thread_created = false, zap_seen = false, completed = false; + int err; + + if (!require_fentry_target(__func__, "zap_vma_range")) + return; + + skel = setup_arena(&addr, true); + if (!skel) + return; + + /* Map the page into user space, so that the free has to zap it. */ + if (!ASSERT_EQ(READ_ONCE(*addr), skel->rodata->old_marker, "old_marker")) + goto out; + + /* Pause the free after its TLB flush, before it releases the range. */ + skel->bss->pause_on_zap = 1; + + if (deferred) { + /* + * Test the nonsleepable free path that gets + * deferred to a worker in the kernel. We do + * so by triggering the arena operation from + * a nonsleepable tracepoint context. + */ + skel->bss->trigger_pid_tgid = + ((__u64)getpid() << 32) | (__u32)sys_gettid(); + skel->bss->trigger_syscall = SYS_getpgid; + skel->bss->deferred_free = 1; + if (!ASSERT_GE(syscall(SYS_getpgid, 0), 0, "deferred_free")) + goto release; + } else { + /* + * Test the sleepable arena free path through a + * syscall test prog. + */ + ctx.skel = skel; + err = pthread_create(&thread, NULL, run_free_thread, &ctx); + if (!ASSERT_OK(err, "pthread_create")) { + skel->bss->release = 1; + goto out; + } + thread_created = true; + } + + /* Wait until the free reaches the user zap. */ + err = wait_for(&skel->bss->zap_entered); + if (!ASSERT_OK(err, "zap_entered")) + goto release; + + zap_seen = true; + + /* The range must stay allocated until the free completes. */ + run_prog(skel->progs.try_realloc, "realloc_during_free"); + ASSERT_NULL(skel->bss->realloc_ptr, "realloc_during_free"); + +release: + skel->bss->release = 1; + if (thread_created) { + ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); + thread_created = false; + completed = ASSERT_OK(ctx.err, "free_run") && + ASSERT_OK(ctx.retval, "free_retval"); + } else if (skel->bss->target_tid) { + err = wait_for(&skel->bss->worker_exited); + completed = ASSERT_OK(err, "worker_exited"); + } + ASSERT_FALSE(skel->bss->timed_out, "zap_timed_out"); + + if (zap_seen && completed && + !run_prog(skel->progs.try_realloc, "realloc_after_free")) { + ASSERT_EQ((unsigned long)skel->bss->realloc_ptr, + (unsigned long)addr, "realloc_after_free"); + ASSERT_EQ(*addr, skel->rodata->new_marker, "new_marker"); + } +out: + arena_race__destroy(skel); +} + +/* + * Force a race between a faulting thread in userspace and a + * free operation on the arena. + */ +static void test_fault_free_realloc(void) +{ + struct sigaction sigbus = { .sa_handler = reader_sigbus }, old_sigbus; + struct fault_thread_ctx fault = {}; + struct arena_race *skel; + pthread_t fault_thread; + bool fault_created = false; + __u64 *addr; + __u64 expected, value; + int err, i; + + skel = setup_arena(&addr, false); + if (!skel) + return; + + skel->bss->realloc_after_free = 1; + + if (!ASSERT_OK(sigaction(SIGBUS, &sigbus, &old_sigbus), "sigaction_sigbus")) + goto out_destroy; + fault.addr = addr; + err = pthread_create(&fault_thread, NULL, fault_reader_thread, &fault); + if (!ASSERT_OK(err, "pthread_create_fault")) + goto out; + fault_created = true; + + for (i = 0; i < 1000; i++) { + LIBBPF_OPTS(bpf_test_run_opts, opts); + + /* Keep deferred frees from earlier iterations on separate slots. */ + if (i) { + skel->bss->page_idx = i; + if (run_prog(skel->progs.alloc_old, "alloc_next")) + goto out; + addr = (__u64 *)(unsigned long)skel->bss->ptr; + __atomic_store_n(&fault.addr, addr, __ATOMIC_RELEASE); + } + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.free_page), + &opts); + if (err) { + ASSERT_OK(err, "free_realloc"); + break; + } + if (opts.retval) { + ASSERT_OK(opts.retval, "free_realloc"); + break; + } + /* The reader may populate the slot first, or free may defer. */ + if (!skel->bss->realloc_ptr) + continue; + value = *addr; + expected = skel->bss->current_marker; + if (value != expected) { + ASSERT_EQ(value, expected, "marker_after_realloc"); + break; + } + } + + WRITE_ONCE(fault.stop, 1); + if (fault_created) { + ASSERT_OK(pthread_join(fault_thread, NULL), "pthread_join_fault"); + fault_created = false; + } +out: + WRITE_ONCE(fault.stop, 1); + if (fault_created) + pthread_join(fault_thread, NULL); + sigaction(SIGBUS, &old_sigbus, NULL); +out_destroy: + arena_race__destroy(skel); +} + +void serial_test_arena_race(void) +{ + cpu_set_t cpuset; + int err; + + err = sched_getaffinity(0, sizeof(cpuset), &cpuset); + if (!ASSERT_OK(err, "sched_getaffinity")) + return; + if (CPU_COUNT(&cpuset) < 2) { + printf("%s:SKIP: at least two runnable CPUs are required\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("realloc_during_free")) + test_realloc_during_free(false); + if (test__start_subtest("realloc_during_deferred_free")) + test_realloc_during_free(true); + if (test__start_subtest("fault_free_realloc")) + test_fault_free_realloc(); +} diff --git a/tools/testing/selftests/bpf/progs/arena_race.c b/tools/testing/selftests/bpf/progs/arena_race.c new file mode 100644 index 000000000000..54dbd662ca70 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_race.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include + +const volatile __u64 old_marker = 0x1111222233334444ULL; +const volatile __u64 new_marker = 0x5555666677778888ULL; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1024); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); +#else + __ulong(map_extra, 0x1ull << 44); +#endif +} arena SEC(".maps"); + +bool skip; +int page_idx; + +void __arena *ptr; +void __arena *realloc_ptr; +bool realloc_after_free; +__u64 current_marker; +__u64 marker_seq; + +int target_tid; +int pause_on_zap; +int zap_entered; +int release; +int timed_out; + +__u64 trigger_pid_tgid; +long trigger_syscall; +int deferred_free; +int worker_armed; +int worker_exited; + +static __always_inline void wait_for_release(void) +{ + while (!*(volatile int *)&release && can_loop) + ; + if (!*(volatile int *)&release) + timed_out = 1; +} + +SEC("syscall") +int alloc_old(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + char __arena *base = arena_base(&arena); + + realloc_ptr = NULL; + ptr = bpf_arena_alloc_pages(&arena, base + (page_idx + 1) * __PAGE_SIZE, 1, + NUMA_NO_NODE, 0); + if (!ptr) + return 1; + p = (__u64 __arena *)ptr; + *p = old_marker; + current_marker = old_marker; +#else + skip = true; +#endif + return 0; +} + +SEC("syscall") +int free_page(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + __u64 marker; + + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, ptr, 1); + if (!realloc_after_free) + return 0; + + marker = new_marker + ++marker_seq; + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (!realloc_ptr) + return 0; + ptr = realloc_ptr; + p = (__u64 __arena *)ptr; + *p = marker; + current_marker = marker; +#endif + return 0; +} + +SEC("syscall") +int try_realloc(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (realloc_ptr) { + ptr = realloc_ptr; + p = (__u64 __arena *)realloc_ptr; + *p = new_marker; + current_marker = new_marker; + } +#endif + return 0; +} + +SEC("tp_btf/sys_enter") +int BPF_PROG(deferred_free_prog, struct pt_regs *regs, long id) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + if (!deferred_free || bpf_get_current_pid_tgid() != trigger_pid_tgid || + id != trigger_syscall) + return 0; + + deferred_free = 0; + /* The worker can run on another CPU before the kfunc returns. */ + worker_armed = 1; + bpf_arena_free_pages(&arena, ptr, 1); +#endif + return 0; +} + +SEC("fentry/arena_free_worker") +int BPF_PROG(trace_free_worker, struct work_struct *work) +{ + if (worker_armed && !target_tid) + target_tid = (__u32)bpf_get_current_pid_tgid(); + return 0; +} + +SEC("fexit/arena_free_worker") +int BPF_PROG(trace_free_worker_ret, struct work_struct *work) +{ + if ((__u32)bpf_get_current_pid_tgid() == target_tid) + worker_exited = 1; + return 0; +} + +/* + * Pause a free while it zaps user mappings: after it cleared the kernel PTEs + * and flushed the TLB, but before it releases the range. zap_vma_range() is + * generic MM code, so it is an fentry target even where + * flush_tlb_kernel_range() is inline (arm64). + */ +SEC("?fentry/zap_vma_range") +int BPF_PROG(trace_zap, struct vm_area_struct *vma, unsigned long address, + unsigned long size) +{ + if ((__u32)bpf_get_current_pid_tgid() != target_tid) + return 0; + zap_entered = 1; + if (pause_on_zap) + wait_for_release(); + return 0; +} + +char _license[] SEC("license") = "GPL"; From f25f6319e248034f3df83352e450ab3172eece43 Mon Sep 17 00:00:00 2001 From: Nicholas Velten Date: Mon, 19 Oct 2026 14:38:20 -0300 Subject: [PATCH 644/660] bpf: Annotate data races on bpf_lru_node type KCSAN reports a data race on bpf_lru_node::type between the LRU list rotation and the free path of the common LRU: BUG: KCSAN: data-race in __bpf_lru_list_rotate / bpf_lru_push_free write to 0xffff88811b758022 of 1 bytes by task 4756 on cpu 1: __bpf_lru_node_move __bpf_lru_list_rotate_active __bpf_lru_list_rotate bpf_lru_list_pop_free_to_local bpf_common_lru_pop_free bpf_lru_pop_free htab_lru_map_update_elem read to 0xffff88811b758022 of 1 bytes by task 4752 on cpu 0: bpf_lru_list_push_free bpf_common_lru_push_free bpf_lru_push_free htab_lru_push_free htab_lru_map_update_elem value changed: 0x00 -> 0x01 bpf_lru_list_push_free() reads node->type for a WARN_ON_ONCE() sanity check before taking l->lock. __bpf_lru_node_move(), called right after under l->lock, does the same check, so drop the unlocked one. bpf_common_lru_push_free() re-checks node->type under loc_l->lock after the lockless READ_ONCE() at function entry. By then the node has been removed from the htab, so only flush and rotate can change its type, through __bpf_lru_node_move_in() and __bpf_lru_node_move() under l->lock. Mark that re-check with READ_ONCE() and those two writers with WRITE_ONCE(), following commit 6df8fb83301d ("bpf_lru_list: Read double-checked variable once without lock"). The remaining writers operate on a node with a single owner that nobody else reads and are left alone. Tested on bpf-next with CONFIG_KCSAN=y in a 4-vCPU virtme-ng guest, using a reproducer that runs 8 threads issuing BPF_MAP_UPDATE_BATCH on a 64-entry BPF_MAP_TYPE_LRU_HASH with overlapping keys. Reported-by: syzbot+1ea4f564bf6f1f3d1dc4@syzkaller.appspotmail.com Signed-off-by: Nicholas Velten Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019-fix-lru-node-type-kcsan-v2-1-c10a410b3565@gmail.com Closes: https://syzkaller.appspot.com/bug?extid=1ea4f564bf6f1f3d1dc4 --- kernel/bpf/bpf_lru_list.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index 5ed7cb4b98c0..3858886b0232 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -71,7 +71,7 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, return; bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); bpf_lru_node_clear_ref(node); /* Reset pending_free only when moving to the free list */ if (tgt_type == BPF_LRU_LIST_T_FREE) @@ -94,7 +94,7 @@ static void __bpf_lru_node_move(struct bpf_lru_list *l, if (node->type != tgt_type) { bpf_lru_list_count_dec(l, node->type); bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); } bpf_lru_node_clear_ref(node); @@ -298,9 +298,6 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, { unsigned long flags; - if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) - return; - if (raw_res_spin_lock_irqsave(&l->lock, flags)) { WRITE_ONCE(node->pending_free, 1); return; @@ -542,7 +539,7 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, return; } - if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { + if (unlikely(READ_ONCE(node->type) != BPF_LRU_LOCAL_LIST_T_PENDING)) { raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); goto check_lru_list; From b71326388a8a24674846c1d8d3b6dd831d9843a9 Mon Sep 17 00:00:00 2001 From: Xia Yan Date: Tue, 20 Oct 2026 17:51:24 +0800 Subject: [PATCH 645/660] tools/bpf: Remove legacy BPF tools Remove bpf_asm, bpf_dbg and bpf_jit_disasm, along with the assembler's lexer and parser, as suggested by Daniel. These tools are old classic BPF and these days there is no need to keep them in kernel anymore. Drop their build and install rules, feature checks and ignore entries. Remove the tool-specific documentation and update the PTP classifier and packet socket test comments. Keep the classic BPF instruction descriptions and assembly examples. Keep the bpftool and resolve_btfids build and clean targets, and the bpftool install target. Suggested-by: Daniel Borkmann Signed-off-by: Xia Yan Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/61f7315e-c835-425e-a99b-d2470ba9a875@iogearbox.net Link: https://lore.kernel.org/bpf/20261020095124.928021-1-yanxia@cmss.chinamobile.com --- Documentation/networking/filter.rst | 230 +--- net/core/ptp_classifier.c | 8 +- tools/bpf/.gitignore | 7 - tools/bpf/Makefile | 101 +- tools/bpf/bpf_asm.c | 52 - tools/bpf/bpf_dbg.c | 1398 ----------------------- tools/bpf/bpf_exp.l | 198 ---- tools/bpf/bpf_exp.y | 668 ----------- tools/bpf/bpf_jit_disasm.c | 334 ------ tools/testing/selftests/net/psock_lib.h | 2 +- 10 files changed, 16 insertions(+), 2982 deletions(-) delete mode 100644 tools/bpf/.gitignore delete mode 100644 tools/bpf/bpf_asm.c delete mode 100644 tools/bpf/bpf_dbg.c delete mode 100644 tools/bpf/bpf_exp.l delete mode 100644 tools/bpf/bpf_exp.y delete mode 100644 tools/bpf/bpf_jit_disasm.c diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index e0b9dc14a9bf..7f95c3199748 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst @@ -183,12 +183,9 @@ access to BPF code as well. BPF engine and instruction set ------------------------------ -Under tools/bpf/ there's a small helper tool called bpf_asm which can -be used to write low-level filters for example scenarios mentioned in the -previous section. Asm-like syntax mentioned here has been implemented in -bpf_asm and will be used for further explanations (instead of dealing with -less readable opcodes directly, principles are the same). The syntax is -closely modelled after Steven McCanne's and Van Jacobson's BPF paper. +The examples below use an assembly notation instead of less readable +opcodes. The notation is closely modelled after Steven McCanne's and +Van Jacobson's BPF paper. The BPF architecture consists of the following basic elements: @@ -201,8 +198,8 @@ The BPF architecture consists of the following basic elements: store", addressable from 0 to 15 ======= ==================================================== -A program, that is translated by bpf_asm into "opcodes" is an array that -consists of the following elements (as already mentioned):: +A BPF program is an array of instructions with the following elements +(as already mentioned):: op:16, jt:8, jf:8, k:32 @@ -213,9 +210,8 @@ contains a miscellaneous argument that can be interpreted in different ways depending on the given instruction in op. The instruction set consists of load, store, branch, alu, miscellaneous -and return instructions that are also represented in bpf_asm syntax. This -table lists all bpf_asm instructions available resp. what their underlying -opcodes as defined in linux/filter.h stand for: +and return instructions. This table lists the assembly notation and what +the corresponding opcodes defined in linux/filter.h do: =========== =================== ===================== Instruction Addressing mode Description @@ -374,141 +370,14 @@ Examples for low-level BPF extension: ret #-1 drop: ret #0 -The above example code can be placed into a file (here called "foo"), and -then be passed to the bpf_asm tool for generating opcodes, output that xt_bpf -and cls_bpf understands and can directly be loaded with. Example with above -ARP code:: +The ARP filter shown above corresponds to the following sock_filter +entries:: - $ ./bpf_asm foo - 4,40 0 0 12,21 0 1 2054,6 0 0 4294967295,6 0 0 0, - -In copy and paste C-like output:: - - $ ./bpf_asm -c foo { 0x28, 0, 0, 0x0000000c }, { 0x15, 0, 1, 0x00000806 }, { 0x06, 0, 0, 0xffffffff }, { 0x06, 0, 0, 0000000000 }, -In particular, as usage with xt_bpf or cls_bpf can result in more complex BPF -filters that might not be obvious at first, it's good to test filters before -attaching to a live system. For that purpose, there's a small tool called -bpf_dbg under tools/bpf/ in the kernel source directory. This debugger allows -for testing BPF filters against given pcap files, single stepping through the -BPF code on the pcap's packets and to do BPF machine register dumps. - -Starting bpf_dbg is trivial and just requires issuing:: - - # ./bpf_dbg - -In case input and output do not equal stdin/stdout, bpf_dbg takes an -alternative stdin source as a first argument, and an alternative stdout -sink as a second one, e.g. `./bpf_dbg test_in.txt test_out.txt`. - -Other than that, a particular libreadline configuration can be set via -file "~/.bpf_dbg_init" and the command history is stored in the file -"~/.bpf_dbg_history". - -Interaction in bpf_dbg happens through a shell that also has auto-completion -support (follow-up example commands starting with '>' denote bpf_dbg shell). -The usual workflow would be to ... - -* load bpf 6,40 0 0 12,21 0 3 2048,48 0 0 23,21 0 1 1,6 0 0 65535,6 0 0 0 - Loads a BPF filter from standard output of bpf_asm, or transformed via - e.g. ``tcpdump -iem1 -ddd port 22 | tr '\n' ','``. Note that for JIT - debugging (next section), this command creates a temporary socket and - loads the BPF code into the kernel. Thus, this will also be useful for - JIT developers. - -* load pcap foo.pcap - - Loads standard tcpdump pcap file. - -* run [] - -bpf passes:1 fails:9 - Runs through all packets from a pcap to account how many passes and fails - the filter will generate. A limit of packets to traverse can be given. - -* disassemble:: - - l0: ldh [12] - l1: jeq #0x800, l2, l5 - l2: ldb [23] - l3: jeq #0x1, l4, l5 - l4: ret #0xffff - l5: ret #0 - - Prints out BPF code disassembly. - -* dump:: - - /* { op, jt, jf, k }, */ - { 0x28, 0, 0, 0x0000000c }, - { 0x15, 0, 3, 0x00000800 }, - { 0x30, 0, 0, 0x00000017 }, - { 0x15, 0, 1, 0x00000001 }, - { 0x06, 0, 0, 0x0000ffff }, - { 0x06, 0, 0, 0000000000 }, - - Prints out C-style BPF code dump. - -* breakpoint 0:: - - breakpoint at: l0: ldh [12] - -* breakpoint 1:: - - breakpoint at: l1: jeq #0x800, l2, l5 - - ... - - Sets breakpoints at particular BPF instructions. Issuing a `run` command - will walk through the pcap file continuing from the current packet and - break when a breakpoint is being hit (another `run` will continue from - the currently active breakpoint executing next instructions): - - * run:: - - -- register dump -- - pc: [0] <-- program counter - code: [40] jt[0] jf[0] k[12] <-- plain BPF code of current instruction - curr: l0: ldh [12] <-- disassembly of current instruction - A: [00000000][0] <-- content of A (hex, decimal) - X: [00000000][0] <-- content of X (hex, decimal) - M[0,15]: [00000000][0] <-- folded content of M (hex, decimal) - -- packet dump -- <-- Current packet from pcap (hex) - len: 42 - 0: 00 19 cb 55 55 a4 00 14 a4 43 78 69 08 06 00 01 - 16: 08 00 06 04 00 01 00 14 a4 43 78 69 0a 3b 01 26 - 32: 00 00 00 00 00 00 0a 3b 01 01 - (breakpoint) - > - - * breakpoint:: - - breakpoints: 0 1 - - Prints currently set breakpoints. - -* step [-, +] - - Performs single stepping through the BPF program from the current pc - offset. Thus, on each step invocation, above register dump is issued. - This can go forwards and backwards in time, a plain `step` will break - on the next BPF instruction, thus +1. (No `run` needs to be issued here.) - -* select - - Selects a given packet from the pcap file to continue from. Thus, on - the next `run` or `step`, the BPF program is being evaluated against - the user pre-selected packet. Numbering starts just as in Wireshark - with index 1. - -* quit - - Exits bpf_dbg. - JIT compiler ------------ @@ -540,83 +409,6 @@ setting bpf_jit_enable to 2, since dumping the final JIT image into the kernel l is discouraged and introspection through bpftool (under tools/bpf/bpftool/) is the generally recommended approach instead. -In the kernel source tree under tools/bpf/, there's bpf_jit_disasm for -generating disassembly out of the kernel log's hexdump:: - - # ./bpf_jit_disasm - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + : - 0: push %rbp - 1: mov %rsp,%rbp - 4: sub $0x60,%rsp - 8: mov %rbx,-0x8(%rbp) - c: mov 0x68(%rdi),%r9d - 10: sub 0x6c(%rdi),%r9d - 14: mov 0xd8(%rdi),%r8 - 1b: mov $0xc,%esi - 20: callq 0xffffffffe0ff9442 - 25: cmp $0x800,%eax - 2a: jne 0x0000000000000042 - 2c: mov $0x17,%esi - 31: callq 0xffffffffe0ff945e - 36: cmp $0x1,%eax - 39: jne 0x0000000000000042 - 3b: mov $0xffff,%eax - 40: jmp 0x0000000000000044 - 42: xor %eax,%eax - 44: leaveq - 45: retq - - Issuing option `-o` will "annotate" opcodes to resulting assembler - instructions, which can be very useful for JIT developers: - - # ./bpf_jit_disasm -o - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + : - 0: push %rbp - 55 - 1: mov %rsp,%rbp - 48 89 e5 - 4: sub $0x60,%rsp - 48 83 ec 60 - 8: mov %rbx,-0x8(%rbp) - 48 89 5d f8 - c: mov 0x68(%rdi),%r9d - 44 8b 4f 68 - 10: sub 0x6c(%rdi),%r9d - 44 2b 4f 6c - 14: mov 0xd8(%rdi),%r8 - 4c 8b 87 d8 00 00 00 - 1b: mov $0xc,%esi - be 0c 00 00 00 - 20: callq 0xffffffffe0ff9442 - e8 1d 94 ff e0 - 25: cmp $0x800,%eax - 3d 00 08 00 00 - 2a: jne 0x0000000000000042 - 75 16 - 2c: mov $0x17,%esi - be 17 00 00 00 - 31: callq 0xffffffffe0ff945e - e8 28 94 ff e0 - 36: cmp $0x1,%eax - 83 f8 01 - 39: jne 0x0000000000000042 - 75 07 - 3b: mov $0xffff,%eax - b8 ff ff 00 00 - 40: jmp 0x0000000000000044 - eb 02 - 42: xor %eax,%eax - 31 c0 - 44: leaveq - c9 - 45: retq - c3 - -For BPF JIT developers, bpf_jit_disasm, bpf_asm and bpf_dbg provides a useful -toolchain for developing and testing the kernel's JIT compiler. - BPF kernel internals -------------------- Internally, for the kernel interpreter, a different instruction set @@ -656,8 +448,8 @@ from eBPF instruction set. Testing ------- -Next to the BPF toolchain, the BPF selftests under tools/testing/selftests/bpf -ship instruction tests in test_progs, see jit_insns, which runs the eBPF +The BPF selftests under tools/testing/selftests/bpf ship instruction +tests in test_progs, see jit_insns, which runs the eBPF instructions over a range of operand magnitudes and signs, register combinations, memory offsets and jump offsets against the BPF interpreter or the JIT compiler, depending on the net.core.bpf_jit_enable setting. diff --git a/net/core/ptp_classifier.c b/net/core/ptp_classifier.c index 598041b0499e..939104e25c75 100644 --- a/net/core/ptp_classifier.c +++ b/net/core/ptp_classifier.c @@ -2,12 +2,8 @@ /* PTP classifier */ -/* The below program is the bpf_asm (tools/net/) representation of - * the opcode array in the ptp_filter structure. - * - * For convenience, this can easily be altered and reviewed with - * bpf_asm and bpf_dbg, e.g. `./bpf_asm -c prog` where prog is a - * simple file containing the below program: +/* The below program is the classic BPF assembly representation of + * the opcode array in the ptp_filter structure: * * ldh [12] ; load ethertype * diff --git a/tools/bpf/.gitignore b/tools/bpf/.gitignore deleted file mode 100644 index cf53342175e7..000000000000 --- a/tools/bpf/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -FEATURE-DUMP.bpf -feature -bpf_asm -bpf_dbg -bpf_exp.yacc.* -bpf_jit_disasm diff --git a/tools/bpf/Makefile b/tools/bpf/Makefile index 9c19e81f3c27..a41360ecb2db 100644 --- a/tools/bpf/Makefile +++ b/tools/bpf/Makefile @@ -1,108 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 include ../scripts/Makefile.include -prefix ?= /usr/local - -LEX = flex -YACC = bison -MAKE = make -INSTALL ?= install - -CFLAGS += -Wall -O2 -CFLAGS += -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi \ - -I$(srctree)/tools/include -CFLAGS += $(EXTRA_CFLAGS) - -# This will work when bpf is built in tools env. where srctree -# isn't set and when invoked from selftests build, where srctree -# is set to ".". building_out_of_srctree is undefined for in srctree -# builds -ifeq ($(srctree),) -update_srctree := 1 -endif -ifndef building_out_of_srctree -update_srctree := 1 -endif -ifeq ($(update_srctree),1) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -FEATURE_USER = .bpf -FEATURE_TESTS = libbfd disassembler-four-args disassembler-init-styled -FEATURE_DISPLAY = libbfd - -check_feat := 1 -NON_CHECK_FEAT_TARGETS := clean bpftool_clean resolve_btfids_clean -ifdef MAKECMDGOALS -ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),) - check_feat := 0 -endif -endif - -ifeq ($(check_feat),1) -ifeq ($(FEATURES_DUMP),) -include $(srctree)/tools/build/Makefile.feature -else -include $(FEATURES_DUMP) -endif -endif - -ifeq ($(feature-disassembler-four-args), 1) -CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE -endif -ifeq ($(feature-disassembler-init-styled), 1) -CFLAGS += -DDISASM_INIT_STYLED -endif - -$(OUTPUT)%.yacc.c: $(srctree)/tools/bpf/%.y - $(QUIET_BISON)$(YACC) -o $@ -d $< - -$(OUTPUT)%.lex.c: $(srctree)/tools/bpf/%.l - $(QUIET_FLEX)$(LEX) -o $@ $< - -$(OUTPUT)%.o: $(srctree)/tools/bpf/%.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -$(OUTPUT)%.yacc.o: $(OUTPUT)%.yacc.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< -$(OUTPUT)%.lex.o: $(OUTPUT)%.lex.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -PROGS = $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg $(OUTPUT)bpf_asm - -all: $(PROGS) bpftool - -$(OUTPUT)bpf_jit_disasm: CFLAGS += -DPACKAGE='bpf_jit_disasm' -$(OUTPUT)bpf_jit_disasm: $(OUTPUT)bpf_jit_disasm.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lopcodes -lbfd -ldl - -$(OUTPUT)bpf_dbg: $(OUTPUT)bpf_dbg.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lreadline - -$(OUTPUT)bpf_asm: $(OUTPUT)bpf_asm.o $(OUTPUT)bpf_exp.yacc.o $(OUTPUT)bpf_exp.lex.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ - -$(OUTPUT)bpf_exp.lex.c: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.yacc.o: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.lex.o: $(OUTPUT)bpf_exp.lex.c +all: bpftool clean: bpftool_clean resolve_btfids_clean - $(call QUIET_CLEAN, bpf-progs) - $(Q)$(RM) -r -- $(OUTPUT)*.o $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg \ - $(OUTPUT)bpf_asm $(OUTPUT)bpf_exp.yacc.* $(OUTPUT)bpf_exp.lex.* - $(call QUIET_CLEAN, core-gen) - $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpf - $(Q)$(RM) -r -- $(OUTPUT)feature -install: $(PROGS) bpftool_install - $(call QUIET_INSTALL, bpf_jit_disasm) - $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/bin - $(Q)$(INSTALL) $(OUTPUT)bpf_jit_disasm $(DESTDIR)$(prefix)/bin/bpf_jit_disasm - $(call QUIET_INSTALL, bpf_dbg) - $(Q)$(INSTALL) $(OUTPUT)bpf_dbg $(DESTDIR)$(prefix)/bin/bpf_dbg - $(call QUIET_INSTALL, bpf_asm) - $(Q)$(INSTALL) $(OUTPUT)bpf_asm $(DESTDIR)$(prefix)/bin/bpf_asm +install: bpftool_install bpftool: $(call descend,bpftool) diff --git a/tools/bpf/bpf_asm.c b/tools/bpf/bpf_asm.c deleted file mode 100644 index 0063c3c029e7..000000000000 --- a/tools/bpf/bpf_asm.c +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF assembler - * - * Instead of libpcap high-level filter expressions, it can be quite - * useful to define filters in low-level BPF assembler (that is kept - * close to Steven McCanne and Van Jacobson's original BPF paper). - * In particular for BPF JIT implementors, JIT security auditors, or - * just for defining BPF expressions that contain extensions which are - * not supported by compilers. - * - * How to get into it: - * - * 1) read Documentation/networking/filter.rst - * 2) Run `bpf_asm [-c] ` to translate into binary - * blob that is loadable with xt_bpf, cls_bpf et al. Note: -c will - * pretty print a C-like construct. - * - * Copyright 2013 Daniel Borkmann - */ - -#include -#include -#include - -extern void bpf_asm_compile(FILE *fp, bool cstyle); - -int main(int argc, char **argv) -{ - FILE *fp = stdin; - bool cstyle = false; - int i; - - for (i = 1; i < argc; i++) { - if (!strncmp("-c", argv[i], 2)) { - cstyle = true; - continue; - } - - fp = fopen(argv[i], "r"); - if (!fp) { - fp = stdin; - continue; - } - - break; - } - - bpf_asm_compile(fp, cstyle); - - return 0; -} diff --git a/tools/bpf/bpf_dbg.c b/tools/bpf/bpf_dbg.c deleted file mode 100644 index 00e560a17baf..000000000000 --- a/tools/bpf/bpf_dbg.c +++ /dev/null @@ -1,1398 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF debugger - * - * Minimal BPF debugger that mimics the kernel's engine (w/o extensions) - * and allows for single stepping through selected packets from a pcap - * with a provided user filter in order to facilitate verification of a - * BPF program. Besides others, this is useful to verify BPF programs - * before attaching to a live system, and can be used in socket filters, - * cls_bpf, xt_bpf, team driver and e.g. PTP code; in particular when a - * single more complex BPF program is being used. Reasons for a more - * complex BPF program are likely primarily to optimize execution time - * for making a verdict when multiple simple BPF programs are combined - * into one in order to prevent parsing same headers multiple times. - * - * More on how to debug BPF opcodes see Documentation/networking/filter.rst - * which is the main document on BPF. Mini howto for getting started: - * - * 1) `./bpf_dbg` to enter the shell (shell cmds denoted with '>'): - * 2) > load bpf 6,40 0 0 12,21 0 3 20... (output from `bpf_asm` or - * `tcpdump -iem1 -ddd port 22 | tr '\n' ','` to load as filter) - * 3) > load pcap foo.pcap - * 4) > run /disassemble/dump/quit (self-explanatory) - * 5) > breakpoint 2 (sets bp at loaded BPF insns 2, do `run` then; - * multiple bps can be set, of course, a call to `breakpoint` - * w/o args shows currently loaded bps, `breakpoint reset` for - * resetting all breakpoints) - * 6) > select 3 (`run` etc will start from the 3rd packet in the pcap) - * 7) > step [-, +] (performs single stepping through the BPF) - * - * Copyright 2013 Daniel Borkmann - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define TCPDUMP_MAGIC 0xa1b2c3d4 - -#define BPF_LDX_B (BPF_LDX | BPF_B) -#define BPF_LDX_W (BPF_LDX | BPF_W) -#define BPF_JMP_JA (BPF_JMP | BPF_JA) -#define BPF_JMP_JEQ (BPF_JMP | BPF_JEQ) -#define BPF_JMP_JGT (BPF_JMP | BPF_JGT) -#define BPF_JMP_JGE (BPF_JMP | BPF_JGE) -#define BPF_JMP_JSET (BPF_JMP | BPF_JSET) -#define BPF_ALU_ADD (BPF_ALU | BPF_ADD) -#define BPF_ALU_SUB (BPF_ALU | BPF_SUB) -#define BPF_ALU_MUL (BPF_ALU | BPF_MUL) -#define BPF_ALU_DIV (BPF_ALU | BPF_DIV) -#define BPF_ALU_MOD (BPF_ALU | BPF_MOD) -#define BPF_ALU_NEG (BPF_ALU | BPF_NEG) -#define BPF_ALU_AND (BPF_ALU | BPF_AND) -#define BPF_ALU_OR (BPF_ALU | BPF_OR) -#define BPF_ALU_XOR (BPF_ALU | BPF_XOR) -#define BPF_ALU_LSH (BPF_ALU | BPF_LSH) -#define BPF_ALU_RSH (BPF_ALU | BPF_RSH) -#define BPF_MISC_TAX (BPF_MISC | BPF_TAX) -#define BPF_MISC_TXA (BPF_MISC | BPF_TXA) -#define BPF_LD_B (BPF_LD | BPF_B) -#define BPF_LD_H (BPF_LD | BPF_H) -#define BPF_LD_W (BPF_LD | BPF_W) - -#ifndef array_size -# define array_size(x) (sizeof(x) / sizeof((x)[0])) -#endif - -#ifndef __check_format_printf -# define __check_format_printf(pos_fmtstr, pos_fmtargs) \ - __attribute__ ((format (printf, (pos_fmtstr), (pos_fmtargs)))) -#endif - -enum { - CMD_OK, - CMD_ERR, - CMD_EX, -}; - -struct shell_cmd { - const char *name; - int (*func)(char *args); -}; - -struct pcap_filehdr { - uint32_t magic; - uint16_t version_major; - uint16_t version_minor; - int32_t thiszone; - uint32_t sigfigs; - uint32_t snaplen; - uint32_t linktype; -}; - -struct pcap_timeval { - int32_t tv_sec; - int32_t tv_usec; -}; - -struct pcap_pkthdr { - struct pcap_timeval ts; - uint32_t caplen; - uint32_t len; -}; - -struct bpf_regs { - uint32_t A; - uint32_t X; - uint32_t M[BPF_MEMWORDS]; - uint32_t R; - bool Rs; - uint16_t Pc; -}; - -static struct sock_filter bpf_image[BPF_MAXINSNS + 1]; -static unsigned int bpf_prog_len; - -static int bpf_breakpoints[64]; -static struct bpf_regs bpf_regs[BPF_MAXINSNS + 1]; -static struct bpf_regs bpf_curr; -static unsigned int bpf_regs_len; - -static int pcap_fd = -1; -static unsigned int pcap_packet; -static size_t pcap_map_size; -static char *pcap_ptr_va_start, *pcap_ptr_va_curr; - -static const char * const op_table[] = { - [BPF_ST] = "st", - [BPF_STX] = "stx", - [BPF_LD_B] = "ldb", - [BPF_LD_H] = "ldh", - [BPF_LD_W] = "ld", - [BPF_LDX] = "ldx", - [BPF_LDX_B] = "ldxb", - [BPF_JMP_JA] = "ja", - [BPF_JMP_JEQ] = "jeq", - [BPF_JMP_JGT] = "jgt", - [BPF_JMP_JGE] = "jge", - [BPF_JMP_JSET] = "jset", - [BPF_ALU_ADD] = "add", - [BPF_ALU_SUB] = "sub", - [BPF_ALU_MUL] = "mul", - [BPF_ALU_DIV] = "div", - [BPF_ALU_MOD] = "mod", - [BPF_ALU_NEG] = "neg", - [BPF_ALU_AND] = "and", - [BPF_ALU_OR] = "or", - [BPF_ALU_XOR] = "xor", - [BPF_ALU_LSH] = "lsh", - [BPF_ALU_RSH] = "rsh", - [BPF_MISC_TAX] = "tax", - [BPF_MISC_TXA] = "txa", - [BPF_RET] = "ret", -}; - -static __check_format_printf(1, 2) int rl_printf(const char *fmt, ...) -{ - int ret; - va_list vl; - - va_start(vl, fmt); - ret = vfprintf(rl_outstream, fmt, vl); - va_end(vl); - - return ret; -} - -static int matches(const char *cmd, const char *pattern) -{ - int len = strlen(cmd); - - if (len > strlen(pattern)) - return -1; - - return memcmp(pattern, cmd, len); -} - -static void hex_dump(const uint8_t *buf, size_t len) -{ - int i; - - rl_printf("%3u: ", 0); - for (i = 0; i < len; i++) { - if (i && !(i % 16)) - rl_printf("\n%3u: ", i); - rl_printf("%02x ", buf[i]); - } - rl_printf("\n"); -} - -static bool bpf_prog_loaded(void) -{ - if (bpf_prog_len == 0) - rl_printf("no bpf program loaded!\n"); - - return bpf_prog_len > 0; -} - -static void bpf_disasm(const struct sock_filter f, unsigned int i) -{ - const char *op, *fmt; - int val = f.k; - char buf[256]; - - switch (f.code) { - case BPF_RET | BPF_K: - op = op_table[BPF_RET]; - fmt = "#%#x"; - break; - case BPF_RET | BPF_A: - op = op_table[BPF_RET]; - fmt = "a"; - break; - case BPF_RET | BPF_X: - op = op_table[BPF_RET]; - fmt = "x"; - break; - case BPF_MISC_TAX: - op = op_table[BPF_MISC_TAX]; - fmt = ""; - break; - case BPF_MISC_TXA: - op = op_table[BPF_MISC_TXA]; - fmt = ""; - break; - case BPF_ST: - op = op_table[BPF_ST]; - fmt = "M[%d]"; - break; - case BPF_STX: - op = op_table[BPF_STX]; - fmt = "M[%d]"; - break; - case BPF_LD_W | BPF_ABS: - op = op_table[BPF_LD_W]; - fmt = "[%d]"; - break; - case BPF_LD_H | BPF_ABS: - op = op_table[BPF_LD_H]; - fmt = "[%d]"; - break; - case BPF_LD_B | BPF_ABS: - op = op_table[BPF_LD_B]; - fmt = "[%d]"; - break; - case BPF_LD_W | BPF_LEN: - op = op_table[BPF_LD_W]; - fmt = "#len"; - break; - case BPF_LD_W | BPF_IND: - op = op_table[BPF_LD_W]; - fmt = "[x+%d]"; - break; - case BPF_LD_H | BPF_IND: - op = op_table[BPF_LD_H]; - fmt = "[x+%d]"; - break; - case BPF_LD_B | BPF_IND: - op = op_table[BPF_LD_B]; - fmt = "[x+%d]"; - break; - case BPF_LD | BPF_IMM: - op = op_table[BPF_LD_W]; - fmt = "#%#x"; - break; - case BPF_LDX | BPF_IMM: - op = op_table[BPF_LDX]; - fmt = "#%#x"; - break; - case BPF_LDX_B | BPF_MSH: - op = op_table[BPF_LDX_B]; - fmt = "4*([%d]&0xf)"; - break; - case BPF_LD | BPF_MEM: - op = op_table[BPF_LD_W]; - fmt = "M[%d]"; - break; - case BPF_LDX | BPF_MEM: - op = op_table[BPF_LDX]; - fmt = "M[%d]"; - break; - case BPF_JMP_JA: - op = op_table[BPF_JMP_JA]; - fmt = "%d"; - val = i + 1 + f.k; - break; - case BPF_JMP_JGT | BPF_X: - op = op_table[BPF_JMP_JGT]; - fmt = "x"; - break; - case BPF_JMP_JGT | BPF_K: - op = op_table[BPF_JMP_JGT]; - fmt = "#%#x"; - break; - case BPF_JMP_JGE | BPF_X: - op = op_table[BPF_JMP_JGE]; - fmt = "x"; - break; - case BPF_JMP_JGE | BPF_K: - op = op_table[BPF_JMP_JGE]; - fmt = "#%#x"; - break; - case BPF_JMP_JEQ | BPF_X: - op = op_table[BPF_JMP_JEQ]; - fmt = "x"; - break; - case BPF_JMP_JEQ | BPF_K: - op = op_table[BPF_JMP_JEQ]; - fmt = "#%#x"; - break; - case BPF_JMP_JSET | BPF_X: - op = op_table[BPF_JMP_JSET]; - fmt = "x"; - break; - case BPF_JMP_JSET | BPF_K: - op = op_table[BPF_JMP_JSET]; - fmt = "#%#x"; - break; - case BPF_ALU_NEG: - op = op_table[BPF_ALU_NEG]; - fmt = ""; - break; - case BPF_ALU_LSH | BPF_X: - op = op_table[BPF_ALU_LSH]; - fmt = "x"; - break; - case BPF_ALU_LSH | BPF_K: - op = op_table[BPF_ALU_LSH]; - fmt = "#%d"; - break; - case BPF_ALU_RSH | BPF_X: - op = op_table[BPF_ALU_RSH]; - fmt = "x"; - break; - case BPF_ALU_RSH | BPF_K: - op = op_table[BPF_ALU_RSH]; - fmt = "#%d"; - break; - case BPF_ALU_ADD | BPF_X: - op = op_table[BPF_ALU_ADD]; - fmt = "x"; - break; - case BPF_ALU_ADD | BPF_K: - op = op_table[BPF_ALU_ADD]; - fmt = "#%d"; - break; - case BPF_ALU_SUB | BPF_X: - op = op_table[BPF_ALU_SUB]; - fmt = "x"; - break; - case BPF_ALU_SUB | BPF_K: - op = op_table[BPF_ALU_SUB]; - fmt = "#%d"; - break; - case BPF_ALU_MUL | BPF_X: - op = op_table[BPF_ALU_MUL]; - fmt = "x"; - break; - case BPF_ALU_MUL | BPF_K: - op = op_table[BPF_ALU_MUL]; - fmt = "#%d"; - break; - case BPF_ALU_DIV | BPF_X: - op = op_table[BPF_ALU_DIV]; - fmt = "x"; - break; - case BPF_ALU_DIV | BPF_K: - op = op_table[BPF_ALU_DIV]; - fmt = "#%d"; - break; - case BPF_ALU_MOD | BPF_X: - op = op_table[BPF_ALU_MOD]; - fmt = "x"; - break; - case BPF_ALU_MOD | BPF_K: - op = op_table[BPF_ALU_MOD]; - fmt = "#%d"; - break; - case BPF_ALU_AND | BPF_X: - op = op_table[BPF_ALU_AND]; - fmt = "x"; - break; - case BPF_ALU_AND | BPF_K: - op = op_table[BPF_ALU_AND]; - fmt = "#%#x"; - break; - case BPF_ALU_OR | BPF_X: - op = op_table[BPF_ALU_OR]; - fmt = "x"; - break; - case BPF_ALU_OR | BPF_K: - op = op_table[BPF_ALU_OR]; - fmt = "#%#x"; - break; - case BPF_ALU_XOR | BPF_X: - op = op_table[BPF_ALU_XOR]; - fmt = "x"; - break; - case BPF_ALU_XOR | BPF_K: - op = op_table[BPF_ALU_XOR]; - fmt = "#%#x"; - break; - default: - op = "nosup"; - fmt = "%#x"; - val = f.code; - break; - } - - memset(buf, 0, sizeof(buf)); - snprintf(buf, sizeof(buf), fmt, val); - buf[sizeof(buf) - 1] = 0; - - if ((BPF_CLASS(f.code) == BPF_JMP && BPF_OP(f.code) != BPF_JA)) - rl_printf("l%d:\t%s %s, l%d, l%d\n", i, op, buf, - i + 1 + f.jt, i + 1 + f.jf); - else - rl_printf("l%d:\t%s %s\n", i, op, buf); -} - -static void bpf_dump_curr(struct bpf_regs *r, struct sock_filter *f) -{ - int i, m = 0; - - rl_printf("pc: [%u]\n", r->Pc); - rl_printf("code: [%u] jt[%u] jf[%u] k[%u]\n", - f->code, f->jt, f->jf, f->k); - rl_printf("curr: "); - bpf_disasm(*f, r->Pc); - - if (f->jt || f->jf) { - rl_printf("jt: "); - bpf_disasm(*(f + f->jt + 1), r->Pc + f->jt + 1); - rl_printf("jf: "); - bpf_disasm(*(f + f->jf + 1), r->Pc + f->jf + 1); - } - - rl_printf("A: [%#08x][%u]\n", r->A, r->A); - rl_printf("X: [%#08x][%u]\n", r->X, r->X); - if (r->Rs) - rl_printf("ret: [%#08x][%u]!\n", r->R, r->R); - - for (i = 0; i < BPF_MEMWORDS; i++) { - if (r->M[i]) { - m++; - rl_printf("M[%d]: [%#08x][%u]\n", i, r->M[i], r->M[i]); - } - } - if (m == 0) - rl_printf("M[0,%d]: [%#08x][%u]\n", BPF_MEMWORDS - 1, 0, 0); -} - -static void bpf_dump_pkt(uint8_t *pkt, uint32_t pkt_caplen, uint32_t pkt_len) -{ - if (pkt_caplen != pkt_len) - rl_printf("cap: %u, len: %u\n", pkt_caplen, pkt_len); - else - rl_printf("len: %u\n", pkt_len); - - hex_dump(pkt, pkt_caplen); -} - -static void bpf_disasm_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - bpf_disasm(f[i], i); -} - -static void bpf_dump_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - rl_printf("/* { op, jt, jf, k }, */\n"); - for (i = 0; i < len; i++) - rl_printf("{ %#04x, %2u, %2u, %#010x },\n", - f[i].code, f[i].jt, f[i].jf, f[i].k); -} - -static bool bpf_runnable(struct sock_filter *f, unsigned int len) -{ - int sock, ret, i; - struct sock_fprog bpf = { - .filter = f, - .len = len, - }; - - sock = socket(AF_INET, SOCK_DGRAM, 0); - if (sock < 0) { - rl_printf("cannot open socket!\n"); - return false; - } - ret = setsockopt(sock, SOL_SOCKET, SO_ATTACH_FILTER, &bpf, sizeof(bpf)); - close(sock); - if (ret < 0) { - rl_printf("program not allowed to run by kernel!\n"); - return false; - } - for (i = 0; i < len; i++) { - if (BPF_CLASS(f[i].code) == BPF_LD && - f[i].k > SKF_AD_OFF) { - rl_printf("extensions currently not supported!\n"); - return false; - } - } - - return true; -} - -static void bpf_reset_breakpoints(void) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) - bpf_breakpoints[i] = -1; -} - -static void bpf_set_breakpoints(unsigned int where) -{ - int i; - bool set = false; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] == (int) where) { - rl_printf("breakpoint already set!\n"); - set = true; - break; - } - - if (bpf_breakpoints[i] == -1 && set == false) { - bpf_breakpoints[i] = where; - set = true; - } - } - - if (!set) - rl_printf("too many breakpoints set, reset first!\n"); -} - -static void bpf_dump_breakpoints(void) -{ - int i; - - rl_printf("breakpoints: "); - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - rl_printf("%d ", bpf_breakpoints[i]); - } - - rl_printf("\n"); -} - -static void bpf_reset(void) -{ - bpf_regs_len = 0; - - memset(bpf_regs, 0, sizeof(bpf_regs)); - memset(&bpf_curr, 0, sizeof(bpf_curr)); -} - -static void bpf_safe_regs(void) -{ - memcpy(&bpf_regs[bpf_regs_len++], &bpf_curr, sizeof(bpf_curr)); -} - -static bool bpf_restore_regs(int off) -{ - unsigned int index = bpf_regs_len - 1 + off; - - if (index == 0) { - bpf_reset(); - return true; - } else if (index < bpf_regs_len) { - memcpy(&bpf_curr, &bpf_regs[index], sizeof(bpf_curr)); - bpf_regs_len = index; - return true; - } else { - rl_printf("reached bottom of register history stack!\n"); - return false; - } -} - -static uint32_t extract_u32(uint8_t *pkt, uint32_t off) -{ - uint32_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohl(r); -} - -static uint16_t extract_u16(uint8_t *pkt, uint32_t off) -{ - uint16_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohs(r); -} - -static uint8_t extract_u8(uint8_t *pkt, uint32_t off) -{ - return pkt[off]; -} - -static void set_return(struct bpf_regs *r) -{ - r->R = 0; - r->Rs = true; -} - -static void bpf_single_step(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - uint32_t K = f->k; - int d; - - switch (f->code) { - case BPF_RET | BPF_K: - r->R = K; - r->Rs = true; - break; - case BPF_RET | BPF_A: - r->R = r->A; - r->Rs = true; - break; - case BPF_RET | BPF_X: - r->R = r->X; - r->Rs = true; - break; - case BPF_MISC_TAX: - r->X = r->A; - break; - case BPF_MISC_TXA: - r->A = r->X; - break; - case BPF_ST: - r->M[K] = r->A; - break; - case BPF_STX: - r->M[K] = r->X; - break; - case BPF_LD_W | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, K); - else - set_return(r); - break; - case BPF_LD_H | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, K); - else - set_return(r); - break; - case BPF_LD_B | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, K); - else - set_return(r); - break; - case BPF_LD_W | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, r->X + K); - break; - case BPF_LD_H | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LD_B | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LDX_B | BPF_MSH: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) { - r->X = extract_u8(pkt, K); - r->X = (r->X & 0xf) << 2; - } else - set_return(r); - break; - case BPF_LD_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LDX_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LD | BPF_IMM: - r->A = K; - break; - case BPF_LDX | BPF_IMM: - r->X = K; - break; - case BPF_LD | BPF_MEM: - r->A = r->M[K]; - break; - case BPF_LDX | BPF_MEM: - r->X = r->M[K]; - break; - case BPF_JMP_JA: - r->Pc += K; - break; - case BPF_JMP_JGT | BPF_X: - r->Pc += r->A > r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGT | BPF_K: - r->Pc += r->A > K ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_X: - r->Pc += r->A >= r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_K: - r->Pc += r->A >= K ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_X: - r->Pc += r->A == r->X ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_K: - r->Pc += r->A == K ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_X: - r->Pc += r->A & r->X ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_K: - r->Pc += r->A & K ? f->jt : f->jf; - break; - case BPF_ALU_NEG: - r->A = -r->A; - break; - case BPF_ALU_LSH | BPF_X: - r->A <<= r->X; - break; - case BPF_ALU_LSH | BPF_K: - r->A <<= K; - break; - case BPF_ALU_RSH | BPF_X: - r->A >>= r->X; - break; - case BPF_ALU_RSH | BPF_K: - r->A >>= K; - break; - case BPF_ALU_ADD | BPF_X: - r->A += r->X; - break; - case BPF_ALU_ADD | BPF_K: - r->A += K; - break; - case BPF_ALU_SUB | BPF_X: - r->A -= r->X; - break; - case BPF_ALU_SUB | BPF_K: - r->A -= K; - break; - case BPF_ALU_MUL | BPF_X: - r->A *= r->X; - break; - case BPF_ALU_MUL | BPF_K: - r->A *= K; - break; - case BPF_ALU_DIV | BPF_X: - case BPF_ALU_MOD | BPF_X: - if (r->X == 0) { - set_return(r); - break; - } - goto do_div; - case BPF_ALU_DIV | BPF_K: - case BPF_ALU_MOD | BPF_K: - if (K == 0) { - set_return(r); - break; - } -do_div: - switch (f->code) { - case BPF_ALU_DIV | BPF_X: - r->A /= r->X; - break; - case BPF_ALU_DIV | BPF_K: - r->A /= K; - break; - case BPF_ALU_MOD | BPF_X: - r->A %= r->X; - break; - case BPF_ALU_MOD | BPF_K: - r->A %= K; - break; - } - break; - case BPF_ALU_AND | BPF_X: - r->A &= r->X; - break; - case BPF_ALU_AND | BPF_K: - r->A &= K; - break; - case BPF_ALU_OR | BPF_X: - r->A |= r->X; - break; - case BPF_ALU_OR | BPF_K: - r->A |= K; - break; - case BPF_ALU_XOR | BPF_X: - r->A ^= r->X; - break; - case BPF_ALU_XOR | BPF_K: - r->A ^= K; - break; - } -} - -static bool bpf_pc_has_breakpoint(uint16_t pc) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - if (bpf_breakpoints[i] == pc) - return true; - } - - return false; -} - -static bool bpf_handle_breakpoint(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - rl_printf("-- register dump --\n"); - bpf_dump_curr(r, &f[r->Pc]); - rl_printf("-- packet dump --\n"); - bpf_dump_pkt(pkt, pkt_caplen, pkt_len); - rl_printf("(breakpoint)\n"); - return true; -} - -static int bpf_run_all(struct sock_filter *f, uint16_t bpf_len, uint8_t *pkt, - uint32_t pkt_caplen, uint32_t pkt_len) -{ - bool stop = false; - - while (bpf_curr.Rs == false && stop == false) { - bpf_safe_regs(); - - if (bpf_pc_has_breakpoint(bpf_curr.Pc)) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static int bpf_run_stepping(struct sock_filter *f, uint16_t bpf_len, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len, int next) -{ - bool stop = false; - int i = 1; - - while (!bpf_curr.Rs && !stop) { - bpf_safe_regs(); - - if (i++ == next) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static bool pcap_loaded(void) -{ - if (pcap_fd < 0) - rl_printf("no pcap file loaded!\n"); - - return pcap_fd >= 0; -} - -static struct pcap_pkthdr *pcap_curr_pkt(void) -{ - return (void *) pcap_ptr_va_curr; -} - -static bool pcap_next_pkt(void) -{ - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - - if (pcap_ptr_va_curr + sizeof(*hdr) - - pcap_ptr_va_start >= pcap_map_size) - return false; - if (hdr->caplen == 0 || hdr->len == 0 || hdr->caplen > hdr->len) - return false; - if (pcap_ptr_va_curr + sizeof(*hdr) + hdr->caplen - - pcap_ptr_va_start >= pcap_map_size) - return false; - - pcap_ptr_va_curr += (sizeof(*hdr) + hdr->caplen); - return true; -} - -static void pcap_reset_pkt(void) -{ - pcap_ptr_va_curr = pcap_ptr_va_start + sizeof(struct pcap_filehdr); -} - -static int try_load_pcap(const char *file) -{ - struct pcap_filehdr *hdr; - struct stat sb; - int ret; - - pcap_fd = open(file, O_RDONLY); - if (pcap_fd < 0) { - rl_printf("cannot open pcap [%s]!\n", strerror(errno)); - return CMD_ERR; - } - - ret = fstat(pcap_fd, &sb); - if (ret < 0) { - rl_printf("cannot fstat pcap file!\n"); - return CMD_ERR; - } - - if (!S_ISREG(sb.st_mode)) { - rl_printf("not a regular pcap file, duh!\n"); - return CMD_ERR; - } - - pcap_map_size = sb.st_size; - if (pcap_map_size <= sizeof(struct pcap_filehdr)) { - rl_printf("pcap file too small!\n"); - return CMD_ERR; - } - - pcap_ptr_va_start = mmap(NULL, pcap_map_size, PROT_READ, - MAP_SHARED | MAP_LOCKED, pcap_fd, 0); - if (pcap_ptr_va_start == MAP_FAILED) { - rl_printf("mmap of file failed!"); - return CMD_ERR; - } - - hdr = (void *) pcap_ptr_va_start; - if (hdr->magic != TCPDUMP_MAGIC) { - rl_printf("wrong pcap magic!\n"); - return CMD_ERR; - } - - pcap_reset_pkt(); - - return CMD_OK; - -} - -static void try_close_pcap(void) -{ - if (pcap_fd >= 0) { - munmap(pcap_ptr_va_start, pcap_map_size); - close(pcap_fd); - - pcap_ptr_va_start = pcap_ptr_va_curr = NULL; - pcap_map_size = 0; - pcap_packet = 0; - pcap_fd = -1; - } -} - -static int cmd_load_bpf(char *bpf_string) -{ - char sp, *token, separator = ','; - unsigned short bpf_len, i = 0; - struct sock_filter tmp; - - bpf_prog_len = 0; - memset(bpf_image, 0, sizeof(bpf_image)); - - if (sscanf(bpf_string, "%hu%c", &bpf_len, &sp) != 2 || - sp != separator || bpf_len > BPF_MAXINSNS || bpf_len == 0) { - rl_printf("syntax error in head length encoding!\n"); - return CMD_ERR; - } - - token = bpf_string; - while ((token = strchr(token, separator)) && (++token)[0]) { - if (i >= bpf_len) { - rl_printf("program exceeds encoded length!\n"); - return CMD_ERR; - } - - if (sscanf(token, "%hu %hhu %hhu %u,", - &tmp.code, &tmp.jt, &tmp.jf, &tmp.k) != 4) { - rl_printf("syntax error at instruction %d!\n", i); - return CMD_ERR; - } - - bpf_image[i].code = tmp.code; - bpf_image[i].jt = tmp.jt; - bpf_image[i].jf = tmp.jf; - bpf_image[i].k = tmp.k; - - i++; - } - - if (i != bpf_len) { - rl_printf("syntax error exceeding encoded length!\n"); - return CMD_ERR; - } else - bpf_prog_len = bpf_len; - if (!bpf_runnable(bpf_image, bpf_prog_len)) - bpf_prog_len = 0; - - return CMD_OK; -} - -static int cmd_load_pcap(char *file) -{ - char *file_trim, *tmp; - - file_trim = strtok_r(file, " ", &tmp); - if (file_trim == NULL) - return CMD_ERR; - - try_close_pcap(); - - return try_load_pcap(file_trim); -} - -static int cmd_load(char *arg) -{ - char *subcmd, *cont = NULL, *tmp = strdup(arg); - int ret = CMD_OK; - - subcmd = strtok_r(tmp, " ", &cont); - if (subcmd == NULL) - goto out; - if (matches(subcmd, "bpf") == 0) { - bpf_reset(); - bpf_reset_breakpoints(); - - if (!cont) - ret = CMD_ERR; - else - ret = cmd_load_bpf(cont); - } else if (matches(subcmd, "pcap") == 0) { - ret = cmd_load_pcap(cont); - } else { -out: - rl_printf("bpf : load bpf code\n"); - rl_printf("pcap : load pcap file\n"); - ret = CMD_ERR; - } - - free(tmp); - return ret; -} - -static int cmd_step(char *num) -{ - struct pcap_pkthdr *hdr; - int steps, ret; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - steps = strtol(num, NULL, 10); - if (steps == 0 || strlen(num) == 0) - steps = 1; - if (steps < 0) { - if (!bpf_restore_regs(steps)) - return CMD_ERR; - steps = 1; - } - - hdr = pcap_curr_pkt(); - ret = bpf_run_stepping(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len, steps); - if (ret >= 0 || bpf_curr.Rs) { - bpf_reset(); - if (!pcap_next_pkt()) { - rl_printf("(going back to first packet)\n"); - pcap_reset_pkt(); - } else { - rl_printf("(next packet)\n"); - } - } - - return CMD_OK; -} - -static int cmd_select(char *num) -{ - unsigned int which, i; - bool have_next = true; - - if (!pcap_loaded() || strlen(num) == 0) - return CMD_ERR; - - which = strtoul(num, NULL, 10); - if (which == 0) { - rl_printf("packet count starts with 1, clamping!\n"); - which = 1; - } - - pcap_reset_pkt(); - bpf_reset(); - - for (i = 0; i < which && (have_next = pcap_next_pkt()); i++) - /* noop */; - if (!have_next || pcap_curr_pkt() == NULL) { - rl_printf("no packet #%u available!\n", which); - pcap_reset_pkt(); - return CMD_ERR; - } - - return CMD_OK; -} - -static int cmd_breakpoint(char *subcmd) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(subcmd) == 0) - bpf_dump_breakpoints(); - else if (matches(subcmd, "reset") == 0) - bpf_reset_breakpoints(); - else { - unsigned int where = strtoul(subcmd, NULL, 10); - - if (where < bpf_prog_len) { - bpf_set_breakpoints(where); - rl_printf("breakpoint at: "); - bpf_disasm(bpf_image[where], where); - } - } - - return CMD_OK; -} - -static int cmd_run(char *num) -{ - static uint32_t pass, fail; - bool has_limit = true; - int pkts = 0, i = 0; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - pkts = strtol(num, NULL, 10); - if (pkts == 0 || strlen(num) == 0) - has_limit = false; - - do { - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - int ret = bpf_run_all(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len); - if (ret > 0) - pass++; - else if (ret == 0) - fail++; - else - return CMD_OK; - bpf_reset(); - } while (pcap_next_pkt() && (!has_limit || (++i < pkts))); - - rl_printf("bpf passes:%u fails:%u\n", pass, fail); - - pcap_reset_pkt(); - bpf_reset(); - - pass = fail = 0; - return CMD_OK; -} - -static int cmd_disassemble(char *line_string) -{ - bool single_line = false; - unsigned long line; - - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(line_string) > 0 && - (line = strtoul(line_string, NULL, 10)) < bpf_prog_len) - single_line = true; - if (single_line) - bpf_disasm(bpf_image[line], line); - else - bpf_disasm_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_dump(char *dontcare) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - - bpf_dump_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_quit(char *dontcare) -{ - return CMD_EX; -} - -static const struct shell_cmd cmds[] = { - { .name = "load", .func = cmd_load }, - { .name = "select", .func = cmd_select }, - { .name = "step", .func = cmd_step }, - { .name = "run", .func = cmd_run }, - { .name = "breakpoint", .func = cmd_breakpoint }, - { .name = "disassemble", .func = cmd_disassemble }, - { .name = "dump", .func = cmd_dump }, - { .name = "quit", .func = cmd_quit }, -}; - -static int execf(char *arg) -{ - char *cmd, *cont, *tmp = strdup(arg); - int i, ret = 0, len; - - cmd = strtok_r(tmp, " ", &cont); - if (cmd == NULL) - goto out; - len = strlen(cmd); - for (i = 0; i < array_size(cmds); i++) { - if (len != strlen(cmds[i].name)) - continue; - if (strncmp(cmds[i].name, cmd, len) == 0) { - ret = cmds[i].func(cont); - break; - } - } -out: - free(tmp); - return ret; -} - -static char *shell_comp_gen(const char *buf, int state) -{ - static int list_index, len; - - if (!state) { - list_index = 0; - len = strlen(buf); - } - - for (; list_index < array_size(cmds); ) { - const char *name = cmds[list_index].name; - - list_index++; - if (strncmp(name, buf, len) == 0) - return strdup(name); - } - - return NULL; -} - -static char **shell_completion(const char *buf, int start, int end) -{ - char **matches = NULL; - - if (start == 0) - matches = rl_completion_matches(buf, shell_comp_gen); - - return matches; -} - -static void intr_shell(int sig) -{ - if (rl_end) - rl_kill_line(-1, 0); - - rl_crlf(); - rl_refresh_line(0, 0); - rl_free_line_state(); -} - -static void init_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - read_history(file); - - rl_instream = fin; - rl_outstream = fout; - - rl_readline_name = "bpf_dbg"; - rl_terminal_name = getenv("TERM"); - - rl_catch_signals = 0; - rl_catch_sigwinch = 1; - - rl_attempted_completion_function = shell_completion; - - rl_bind_key('\t', rl_complete); - - rl_bind_key_in_map('\t', rl_complete, emacs_meta_keymap); - rl_bind_key_in_map('\033', rl_complete, emacs_meta_keymap); - - snprintf(file, sizeof(file), "%s/.bpf_dbg_init", getenv("HOME")); - rl_read_init_file(file); - - rl_prep_terminal(0); - rl_set_signals(); - - signal(SIGINT, intr_shell); -} - -static void exit_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - write_history(file); - - clear_history(); - rl_deprep_terminal(); - - try_close_pcap(); - - if (fin != stdin) - fclose(fin); - if (fout != stdout) - fclose(fout); -} - -static int run_shell_loop(FILE *fin, FILE *fout) -{ - char *buf; - - init_shell(fin, fout); - - while ((buf = readline("> ")) != NULL) { - int ret = execf(buf); - if (ret == CMD_EX) - break; - if (ret == CMD_OK && strlen(buf) > 0) - add_history(buf); - - free(buf); - } - - exit_shell(fin, fout); - return 0; -} - -int main(int argc, char **argv) -{ - FILE *fin = NULL, *fout = NULL; - - if (argc >= 2) - fin = fopen(argv[1], "r"); - if (argc >= 3) - fout = fopen(argv[2], "w"); - - return run_shell_loop(fin ? : stdin, fout ? : stdout); -} diff --git a/tools/bpf/bpf_exp.l b/tools/bpf/bpf_exp.l deleted file mode 100644 index 4da8d053d68f..000000000000 --- a/tools/bpf/bpf_exp.l +++ /dev/null @@ -1,198 +0,0 @@ -/* - * BPF asm code lexer - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include -#include -#include -#include - -#include - -#include "bpf_exp.yacc.h" - -extern void yyerror(const char *str); - -%} - -%option align -%option ecs - -%option nounput -%option noreject -%option noinput -%option noyywrap - -%option 8bit -%option caseless -%option yylineno - -%% - -"ldb" { return OP_LDB; } -"ldh" { return OP_LDH; } -"ld" { return OP_LD; } -"ldi" { return OP_LDI; } -"ldx" { return OP_LDX; } -"ldxi" { return OP_LDXI; } -"ldxb" { return OP_LDXB; } -"st" { return OP_ST; } -"stx" { return OP_STX; } -"jmp" { return OP_JMP; } -"ja" { return OP_JMP; } -"jeq" { return OP_JEQ; } -"jneq" { return OP_JNEQ; } -"jne" { return OP_JNEQ; } -"jlt" { return OP_JLT; } -"jle" { return OP_JLE; } -"jgt" { return OP_JGT; } -"jge" { return OP_JGE; } -"jset" { return OP_JSET; } -"add" { return OP_ADD; } -"sub" { return OP_SUB; } -"mul" { return OP_MUL; } -"div" { return OP_DIV; } -"mod" { return OP_MOD; } -"neg" { return OP_NEG; } -"and" { return OP_AND; } -"xor" { return OP_XOR; } -"or" { return OP_OR; } -"lsh" { return OP_LSH; } -"rsh" { return OP_RSH; } -"ret" { return OP_RET; } -"tax" { return OP_TAX; } -"txa" { return OP_TXA; } - -"#"?("len") { return K_PKT_LEN; } - -"#"?("proto") { - yylval.number = SKF_AD_PROTOCOL; - return extension; - } -"#"?("type") { - yylval.number = SKF_AD_PKTTYPE; - return extension; - } -"#"?("poff") { - yylval.number = SKF_AD_PAY_OFFSET; - return extension; - } -"#"?("ifidx") { - yylval.number = SKF_AD_IFINDEX; - return extension; - } -"#"?("nla") { - yylval.number = SKF_AD_NLATTR; - return extension; - } -"#"?("nlan") { - yylval.number = SKF_AD_NLATTR_NEST; - return extension; - } -"#"?("mark") { - yylval.number = SKF_AD_MARK; - return extension; - } -"#"?("queue") { - yylval.number = SKF_AD_QUEUE; - return extension; - } -"#"?("hatype") { - yylval.number = SKF_AD_HATYPE; - return extension; - } -"#"?("rxhash") { - yylval.number = SKF_AD_RXHASH; - return extension; - } -"#"?("cpu") { - yylval.number = SKF_AD_CPU; - return extension; - } -"#"?("vlan_tci") { - yylval.number = SKF_AD_VLAN_TAG; - return extension; - } -"#"?("vlan_pr") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_avail") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_tpid") { - yylval.number = SKF_AD_VLAN_TPID; - return extension; - } -"#"?("rand") { - yylval.number = SKF_AD_RANDOM; - return extension; - } - -":" { return ':'; } -"," { return ','; } -"#" { return '#'; } -"%" { return '%'; } -"[" { return '['; } -"]" { return ']'; } -"(" { return '('; } -")" { return ')'; } -"x" { return 'x'; } -"a" { return 'a'; } -"+" { return '+'; } -"M" { return 'M'; } -"*" { return '*'; } -"&" { return '&'; } - -([0][x][a-fA-F0-9]+) { - yylval.number = strtoul(yytext, NULL, 16); - return number; - } -([0][b][0-1]+) { - yylval.number = strtol(yytext + 2, NULL, 2); - return number; - } -(([0])|([-+]?[1-9][0-9]*)) { - yylval.number = strtol(yytext, NULL, 10); - return number; - } -([0][0-7]+) { - yylval.number = strtol(yytext + 1, NULL, 8); - return number; - } -[a-zA-Z_][a-zA-Z0-9_]+ { - yylval.label = strdup(yytext); - return label; - } - -"/*"([^\*]|\*[^/])*"*/" { /* NOP */ } -";"[^\n]* { /* NOP */ } -^#.* { /* NOP */ } -[ \t]+ { /* NOP */ } -[ \n]+ { /* NOP */ } - -. { - printf("unknown character \'%s\'", yytext); - yyerror("lex unknown character"); - } - -%% diff --git a/tools/bpf/bpf_exp.y b/tools/bpf/bpf_exp.y deleted file mode 100644 index dfb7254a24e8..000000000000 --- a/tools/bpf/bpf_exp.y +++ /dev/null @@ -1,668 +0,0 @@ -/* - * BPF asm code parser - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bpf_exp.yacc.h" - -enum jmp_type { JTL, JFL, JKL }; - -extern FILE *yyin; -extern int yylineno; -extern int yylex(void); -extern void yyerror(const char *str); - -extern void bpf_asm_compile(FILE *fp, bool cstyle); -static void bpf_set_curr_instr(uint16_t op, uint8_t jt, uint8_t jf, uint32_t k); -static void bpf_set_curr_label(char *label); -static void bpf_set_jmp_label(char *label, enum jmp_type type); - -%} - -%union { - char *label; - uint32_t number; -} - -%token OP_LDB OP_LDH OP_LD OP_LDX OP_ST OP_STX OP_JMP OP_JEQ OP_JGT OP_JGE -%token OP_JSET OP_ADD OP_SUB OP_MUL OP_DIV OP_AND OP_OR OP_XOR OP_LSH OP_RSH -%token OP_RET OP_TAX OP_TXA OP_LDXB OP_MOD OP_NEG OP_JNEQ OP_JLT OP_JLE OP_LDI -%token OP_LDXI - -%token K_PKT_LEN - -%token ':' ',' '[' ']' '(' ')' 'x' 'a' '+' 'M' '*' '&' '#' '%' - -%token extension number label - -%type