#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
#define MAX_BPF_STACK 512
#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)
struct elem {
struct bpf_timer t;
char pad[256];
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__type(key, int);
__type(value, struct elem);
} array SEC(".maps");
SEC("kprobe")
__description("Private stack, single prog")
__success
__arch_x86_64
__jited(" movabsq $0x{{.*}}, %r9")
__jited(" addq %gs:{{.*}}, %r9")
__jited(" movl $0x2a, %edi")
__jited(" movq %rdi, -0x100(%r9)")
__arch_arm64
__jited(" stp x25, x27, [sp, {{.*}}]!")
__jited(" mov x27, {{.*}}")
__jited(" movk x27, {{.*}}, lsl #16")
__jited(" movk x27, {{.*}}")
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
__jited(" mov x0, #0x2a")
__jited(" str x0, [x27]")
__jited("...")
__jited(" ldp x25, x27, [sp], {{.*}}")
__naked void private_stack_single_prog(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - 256) = r1; \
r0 = 0; \
exit; \
" ::: __clobber_all);
}
SEC("raw_tp")
__description("No private stack")
__success
__arch_x86_64
__jited(" subq $0x8, %rsp")
__arch_arm64
__jited(" mov x25, sp")
__jited(" sub sp, sp, #0x10")
__naked void no_private_stack_nested(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - 8) = r1; \
r0 = 0; \
exit; \
" ::: __clobber_all);
}
__used
__naked static void cumulative_stack_depth_subprog(void)
{
asm volatile (" \
r1 = 41; \
*(u64 *)(r10 - 32) = r1; \
call %[bpf_get_smp_processor_id]; \
exit; \
" :
: __imm(bpf_get_smp_processor_id)
: __clobber_all);
}
SEC("kprobe")
__description("Private stack, subtree > MAX_BPF_STACK")
__success
__arch_x86_64
__jited(" movabsq $0x{{.*}}, %r9")
__jited(" addq %gs:{{.*}}, %r9")
__jited(" movl $0x2a, %edi")
__jited(" movq %rdi, -0x200(%r9)")
__jited(" pushq %r9")
__jited(" callq 0x{{.*}}")
__jited(" popq %r9")
__jited(" xorl %eax, %eax")
__arch_arm64
__jited(" stp x25, x27, [sp, {{.*}}]!")
__jited(" mov x27, {{.*}}")
__jited(" movk x27, {{.*}}, lsl #16")
__jited(" movk x27, {{.*}}")
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
__jited(" mov x0, #0x2a")
__jited(" str x0, [x27]")
__jited(" bl {{.*}}")
__jited("...")
__jited(" ldp x25, x27, [sp], {{.*}}")
__naked void private_stack_nested_1(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - %[max_bpf_stack]) = r1; \
call cumulative_stack_depth_subprog; \
r0 = 0; \
exit; \
" :
: __imm_const(max_bpf_stack, MAX_BPF_STACK)
: __clobber_all);
}
__naked __noinline __used
static unsigned long loop_callback(void)
{
asm volatile (" \
call %[bpf_get_prandom_u32]; \
r1 = 42; \
*(u64 *)(r10 - 512) = r1; \
call cumulative_stack_depth_subprog; \
r0 = 0; \
exit; \
" :
: __imm(bpf_get_prandom_u32)
: __clobber_common);
}
SEC("raw_tp")
__description("Private stack, callback")
__success
__arch_x86_64
__jited("func #1")
__jited(" endbr64")
__jited(" nopl (%rax,%rax)")
__jited(" nopl (%rax)")
__jited(" pushq %rbp")
__jited(" movq %rsp, %rbp")
__jited(" endbr64")
__jited(" movabsq $0x{{.*}}, %r9")
__jited(" addq %gs:{{.*}}, %r9")
__jited(" pushq %r9")
__jited(" callq")
__jited(" popq %r9")
__jited(" movl $0x2a, %edi")
__jited(" movq %rdi, -0x200(%r9)")
__jited(" pushq %r9")
__jited(" callq")
__jited(" popq %r9")
__arch_arm64
__jited("func #1")
__jited("...")
__jited(" stp x25, x27, [sp, {{.*}}]!")
__jited(" mov x27, {{.*}}")
__jited(" movk x27, {{.*}}, lsl #16")
__jited(" movk x27, {{.*}}")
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
__jited(" bl 0x{{.*}}")
__jited(" add x7, x0, #0x0")
__jited(" mov x0, #0x2a")
__jited(" str x0, [x27]")
__jited(" bl 0x{{.*}}")
__jited(" add x7, x0, #0x0")
__jited(" mov x7, #0x0")
__jited(" ldp x25, x27, [sp], {{.*}}")
__naked void private_stack_callback(void)
{
asm volatile (" \
r1 = 1; \
r2 = %[loop_callback]; \
r3 = 0; \
r4 = 0; \
call %[bpf_loop]; \
r0 = 0; \
exit; \
" :
: __imm_ptr(loop_callback),
__imm(bpf_loop)
: __clobber_common);
}
SEC("fentry/bpf_fentry_test9")
__description("Private stack, exception in main prog")
__success __retval(0)
__arch_x86_64
__jited(" pushq %r9")
__jited(" callq")
__jited(" popq %r9")
__arch_arm64
__jited(" stp x29, x30, [sp, #-0x10]!")
__jited(" mov x29, sp")
__jited(" stp xzr, x26, [sp, #-0x10]!")
__jited(" mov x26, sp")
__jited(" stp x19, x20, [sp, #-0x10]!")
__jited(" stp x21, x22, [sp, #-0x10]!")
__jited(" stp x23, x24, [sp, #-0x10]!")
__jited(" stp x25, x26, [sp, #-0x10]!")
__jited(" stp x27, x28, [sp, #-0x10]!")
__jited(" mov x27, {{.*}}")
__jited(" movk x27, {{.*}}, lsl #16")
__jited(" movk x27, {{.*}}")
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
__jited(" mov x0, #0x2a")
__jited(" str x0, [x27]")
__jited(" mov x0, #0x0")
__jited(" bl 0x{{.*}}")
__jited(" add x7, x0, #0x0")
__jited(" ldp x27, x28, [sp], #0x10")
int private_stack_exception_main_prog(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - 512) = r1; \
" ::: __clobber_common);
bpf_throw(0);
return 0;
}
__used static int subprog_exception(void)
{
bpf_throw(0);
return 0;
}
SEC("fentry/bpf_fentry_test9")
__description("Private stack, exception in subprog")
__success __retval(0)
__arch_x86_64
__jited(" movq %rdi, -0x200(%r9)")
__jited(" pushq %r9")
__jited(" callq")
__jited(" popq %r9")
__arch_arm64
__jited(" stp x27, x28, [sp, #-0x10]!")
__jited(" mov x27, {{.*}}")
__jited(" movk x27, {{.*}}, lsl #16")
__jited(" movk x27, {{.*}}")
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
__jited(" mov x0, #0x2a")
__jited(" str x0, [x27]")
__jited(" bl 0x{{.*}}")
__jited(" add x7, x0, #0x0")
__jited(" ldp x27, x28, [sp], #0x10")
int private_stack_exception_sub_prog(void)
{
asm volatile (" \
r1 = 42; \
*(u64 *)(r10 - 512) = r1; \
call subprog_exception; \
" ::: __clobber_common);
return 0;
}
int glob;
__noinline static void subprog2(int *val)
{
glob += val[0] * 2;
}
__noinline static void subprog1(int *val)
{
int tmp[64] = {};
tmp[0] = *val;
subprog2(tmp);
}
__noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
{
subprog1(key);
return 0;
}
__noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
{
return 0;
}
SEC("fentry/bpf_fentry_test9")
__description("Private stack, async callback, not nested")
__success __retval(0)
__arch_x86_64
__jited(" movabsq $0x{{.*}}, %r9")
__arch_arm64
__jited(" mrs x10, TPIDR_EL{{[0-1]}}")
__jited(" add x27, x27, x10")
__jited(" add x25, x27, {{.*}}")
int private_stack_async_callback_1(void)
{
struct bpf_timer *arr_timer;
int array_key = 0;
arr_timer = bpf_map_lookup_elem(&array, &array_key);
if (!arr_timer)
return 0;
bpf_timer_init(arr_timer, &array, 1);
bpf_timer_set_callback(arr_timer, timer_cb2);
bpf_timer_start(arr_timer, 0, 0);
subprog1(&array_key);
return 0;
}
SEC("fentry/bpf_fentry_test9")
__description("Private stack, async callback, potential nesting")
__success __retval(0)
__arch_x86_64
__jited(" subq $0x100, %rsp")
__arch_arm64
__jited(" sub sp, sp, #0x100")
int private_stack_async_callback_2(void)
{
struct bpf_timer *arr_timer;
int array_key = 0;
arr_timer = bpf_map_lookup_elem(&array, &array_key);
if (!arr_timer)
return 0;
bpf_timer_init(arr_timer, &array, 1);
bpf_timer_set_callback(arr_timer, timer_cb1);
bpf_timer_start(arr_timer, 0, 0);
subprog1(&array_key);
return 0;
}
#else
SEC("kprobe")
__description("private stack is not supported, use a dummy test")
__success
int dummy_test(void)
{
return 0;
}
#endif
char _license[] SEC("license") = "GPL";
