/* SPDX-License-Identifier: GPL-2.0-only */ /* * arch/arm64/kernel/entry-ftrace.S * * Copyright (C) 2013 Linaro Limited * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> */ #include <linux/linkage.h> #include <linux/cfi_types.h> #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/ftrace.h> #include <asm/insn.h> #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS /* * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before * the regular function prologue. For an enabled callsite, ftrace_init_nop() and * ftrace_make_call() have patched those NOPs to: * * MOV X9, LR * BL ftrace_caller * * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to * clobber. * * We save the callsite's context into a struct ftrace_regs before invoking any * ftrace callbacks. So that we can get a sensible backtrace, we create frame * records for the callsite and the ftrace entry assembly. This is not * sufficient for reliable stacktrace: until we create the callsite stack * record, its caller is missing from the LR and existing chain of frame * records. */ SYM_CODE_START(ftrace_caller) bti c #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS /* * The literal pointer to the ops is at an 8-byte aligned boundary * which is either 12 or 16 bytes before the BL instruction in the call * site. See ftrace_call_adjust() for details. * * Therefore here the LR points at `literal + 16` or `literal + 20`, * and we can find the address of the literal in either case by * aligning to an 8-byte boundary and subtracting 16. We do the * alignment first as this allows us to fold the subtraction into the * LDR. */ bic x11, x30, 0x7 ldr x11, [x11, #-(4 * AARCH64_INSN_SIZE)] // op #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* * If the op has a direct call, handle it immediately without * saving/restoring registers. */ ldr x17, [x11, #FTRACE_OPS_DIRECT_CALL] // op->direct_call cbnz x17, ftrace_caller_direct #endif #endif /* Save original SP */ mov x10, sp /* Make room for ftrace regs, plus two frame records */ sub sp, sp, #(FREGS_SIZE + 32) /* Save function arguments */ stp x0, x1, [sp, #FREGS_X0] stp x2, x3, [sp, #FREGS_X2] stp x4, x5, [sp, #FREGS_X4] stp x6, x7, [sp, #FREGS_X6] str x8, [sp, #FREGS_X8] #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS str xzr, [sp, #FREGS_DIRECT_TRAMP] #endif /* Save the callsite's FP, LR, SP */ str x29, [sp, #FREGS_FP] str x9, [sp, #FREGS_LR] str x10, [sp, #FREGS_SP] /* Save the PC after the ftrace callsite */ str x30, [sp, #FREGS_PC] /* Create a frame record for the callsite above the ftrace regs */ stp x29, x9, [sp, #FREGS_SIZE + 16] add x29, sp, #FREGS_SIZE + 16 /* Create our frame record above the ftrace regs */ stp x29, x30, [sp, #FREGS_SIZE] add x29, sp, #FREGS_SIZE /* Prepare arguments for the tracer func */ sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) mov x1, x9 // parent_ip (callsite's LR) mov x3, sp // regs #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS mov x2, x11 // op ldr x4, [x2, #FTRACE_OPS_FUNC] // op->func blr x4 // op->func(ip, parent_ip, op, regs) #else ldr_l x2, function_trace_op // op SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) bl ftrace_stub // func(ip, parent_ip, op, regs) #endif /* * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved * x19-x29 per the AAPCS, and we created frame records upon entry, so we need * to restore x0-x8, x29, and x30. */ /* Restore function arguments */ ldp x0, x1, [sp, #FREGS_X0] ldp x2, x3, [sp, #FREGS_X2] ldp x4, x5, [sp, #FREGS_X4] ldp x6, x7, [sp, #FREGS_X6] ldr x8, [sp, #FREGS_X8] /* Restore the callsite's FP */ ldr x29, [sp, #FREGS_FP] #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS ldr x17, [sp, #FREGS_DIRECT_TRAMP] cbnz x17, ftrace_caller_direct_late #endif /* Restore the callsite's LR and PC */ ldr x30, [sp, #FREGS_LR] ldr x9, [sp, #FREGS_PC] /* Restore the callsite's SP */ add sp, sp, #FREGS_SIZE + 32 ret x9 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_INNER_LABEL(ftrace_caller_direct_late, SYM_L_LOCAL) /* * Head to a direct trampoline in x17 after having run other tracers. * The ftrace_regs are live, and x0-x8 and FP have been restored. The * LR, PC, and SP have not been restored. */ /* * Restore the callsite's LR and PC matching the trampoline calling * convention. */ ldr x9, [sp, #FREGS_LR] ldr x30, [sp, #FREGS_PC] /* Restore the callsite's SP */ add sp, sp, #FREGS_SIZE + 32 SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL) /* * Head to a direct trampoline in x17. * * We use `BR X17` as this can safely land on a `BTI C` or `PACIASP` in * the trampoline, and will not unbalance any return stack. */ br x17 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ SYM_CODE_END(ftrace_caller) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) bti c mov x10, x30 mov x30, x9 ret x10 SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ #else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ /* * Gcc with -pg will put the following code in the beginning of each function: * mov x0, x30 * bl _mcount * [function's body ...] * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic * ftrace is enabled. * * Please note that x0 as an argument will not be used here because we can * get lr(x30) of instrumented function at any time by winding up call stack * as long as the kernel is compiled without -fomit-frame-pointer. * (or CONFIG_FRAME_POINTER, this is forced on arm64) * * stack layout after mcount_enter in _mcount(): * * current sp/fp => 0:+-----+ * in _mcount() | x29 | -> instrumented function's fp * +-----+ * | x30 | -> _mcount()'s lr (= instrumented function's pc) * old sp => +16:+-----+ * when instrumented | | * function calls | ... | * _mcount() | | * | | * instrumented => +xx:+-----+ * function's fp | x29 | -> parent's fp * +-----+ * | x30 | -> instrumented function's lr (= parent's pc) * +-----+ * | ... | */ .macro mcount_enter stp x29, x30, [sp, #-16]! mov x29, sp .endm .macro mcount_exit ldp x29, x30, [sp], #16 ret .endm .macro mcount_adjust_addr rd, rn sub \rd, \rn, #AARCH64_INSN_SIZE .endm /* for instrumented function's parent */ .macro mcount_get_parent_fp reg ldr \reg, [x29] ldr \reg, [\reg] .endm /* for instrumented function */ .macro mcount_get_pc0 reg mcount_adjust_addr \reg, x30 .endm .macro mcount_get_pc reg ldr \reg, [x29, #8] mcount_adjust_addr \reg, \reg .endm .macro mcount_get_lr reg ldr \reg, [x29] ldr \reg, [\reg, #8] .endm .macro mcount_get_lr_addr reg ldr \reg, [x29] add \reg, \reg, #8 .endm /* * _mcount() is used to build the kernel with -pg option, but all the branch * instructions to _mcount() are replaced to NOP initially at kernel start up, * and later on, NOP to branch to ftrace_caller() when enabled or branch to * NOP when disabled per-function base. */ SYM_FUNC_START(_mcount) ret SYM_FUNC_END(_mcount) EXPORT_SYMBOL(_mcount) NOKPROBE(_mcount) /* * void ftrace_caller(unsigned long return_address) * @return_address: return address to instrumented function * * This function is a counterpart of _mcount() in 'static' ftrace, and * makes calls to: * - tracer function to probe instrumented function's entry, * - ftrace_graph_caller to set up an exit hook */ SYM_FUNC_START(ftrace_caller) mcount_enter mcount_get_pc0 x0 // function's pc mcount_get_lr x1 // function's lr SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr); nop // This will be replaced with "bl xxx" // where xxx can be any kind of tracer. #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); nop // If enabled, this will be replaced // "b ftrace_graph_caller" #endif mcount_exit SYM_FUNC_END(ftrace_caller) #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* * void ftrace_graph_caller(void) * * Called from _mcount() or ftrace_caller() when function_graph tracer is * selected. * This function w/ prepare_ftrace_return() fakes link register's value on * the call stack in order to intercept instrumented function's return path * and run return_to_handler() later on its exit. */ SYM_FUNC_START(ftrace_graph_caller) mcount_get_pc x0 // function's pc mcount_get_lr_addr x1 // pointer to function's saved lr mcount_get_parent_fp x2 // parent's fp bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) mcount_exit SYM_FUNC_END(ftrace_graph_caller) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ SYM_TYPED_FUNC_START(ftrace_stub) ret SYM_FUNC_END(ftrace_stub) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_TYPED_FUNC_START(ftrace_stub_graph) ret SYM_FUNC_END(ftrace_stub_graph) /* * void return_to_handler(void) * * Run ftrace_return_to_handler() before going back to parent. * @fp is checked against the value passed by ftrace_graph_caller(). */ SYM_CODE_START(return_to_handler) /* Make room for ftrace_regs */ sub sp, sp, #FREGS_SIZE /* Save return value regs */ stp x0, x1, [sp, #FREGS_X0] stp x2, x3, [sp, #FREGS_X2] stp x4, x5, [sp, #FREGS_X4] stp x6, x7, [sp, #FREGS_X6] /* Save the callsite's FP */ str x29, [sp, #FREGS_FP] mov x0, sp bl ftrace_return_to_handler // addr = ftrace_return_to_hander(fregs); mov x30, x0 // restore the original return address /* Restore return value regs */ ldp x0, x1, [sp, #FREGS_X0] ldp x2, x3, [sp, #FREGS_X2] ldp x4, x5, [sp, #FREGS_X4] ldp x6, x7, [sp, #FREGS_X6] add sp, sp, #FREGS_SIZE ret SYM_CODE_END(return_to_handler) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
