drivers/net/ethernet/netronome/nfp/bpf/jit.c

3315

if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {

drivers/net/ethernet/netronome/nfp/bpf/jit.c

3391

if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {

drivers/net/ethernet/netronome/nfp/bpf/jit.c

3611

nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) {

drivers/net/ethernet/netronome/nfp/bpf/jit.c

3649

unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;

drivers/net/ethernet/netronome/nfp/bpf/jit.c

3755

if (nfp_prog->subprog[idx].needs_reg_push)

drivers/net/ethernet/netronome/nfp/bpf/jit.c

3837

depth = nfp_prog->subprog[0].stack_depth;

drivers/net/ethernet/netronome/nfp/bpf/main.h

541

struct nfp_bpf_subprog_info *subprog;

drivers/net/ethernet/netronome/nfp/bpf/offload.c

180

kfree(nfp_prog->subprog);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

686

nfp_prog->subprog[index].needs_reg_push = 1;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

714

frame_depths[frame] = nfp_prog->subprog[idx].stack_depth;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

773

nfp_prog->subprog = kzalloc_objs(nfp_prog->subprog[0],

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

775

if (!nfp_prog->subprog)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

782

nfp_prog->subprog[i].stack_depth = info[i].stack_depth;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

788

nfp_prog->subprog[i].stack_depth += REG_WIDTH;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

790

if (nfp_prog->subprog[i].needs_reg_push)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

791

nfp_prog->subprog[i].stack_depth += BPF_REG_SIZE * 4;

include/linux/bpf.h

3024

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);

include/linux/bpf_verifier.h

856

static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

858

return &env->prog->aux->func_info_aux[subprog];

include/linux/bpf_verifier.h

861

static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

863

return &env->subprog_info[subprog];

kernel/bpf/btf.c

7787

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)

kernel/bpf/btf.c

7789

bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL;

kernel/bpf/btf.c

7790

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/btf.c

7808

btf_id = prog->aux->func_info[subprog].type_id;

kernel/bpf/btf.c

7822

subprog);

kernel/bpf/btf.c

7827

if (prog->aux->func_info_aux[subprog].unreliable) {

kernel/bpf/liveness.c

181

struct bpf_subprog_info *subprog;

kernel/bpf/liveness.c

190

subprog = bpf_find_containing_subprog(env, callchain->sp_starts[callchain->curframe]);

kernel/bpf/liveness.c

191

subprog_sz = (subprog + 1)->start - subprog->start;

kernel/bpf/liveness.c

617

struct bpf_subprog_info *subprog;

kernel/bpf/liveness.c

639

subprog = bpf_find_containing_subprog(env, this_subprog_start);

kernel/bpf/liveness.c

640

po_start = subprog->postorder_start;

kernel/bpf/liveness.c

641

po_end = (subprog + 1)->postorder_start;

kernel/bpf/verifier.c

10640

static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,

kernel/bpf/verifier.c

10672

subprog /* subprog number within this prog */);

kernel/bpf/verifier.c

10688

static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

10692

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

10697

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

10789

static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

10800

btf_id = prog->aux->func_info[subprog].type_id;

kernel/bpf/verifier.c

10804

if (prog->aux->func_info_aux[subprog].unreliable)

kernel/bpf/verifier.c

10807

err = btf_check_func_arg_match(env, subprog, btf, regs);

kernel/bpf/verifier.c

10813

prog->aux->func_info_aux[subprog].unreliable = true;

kernel/bpf/verifier.c

10818

int insn_idx, int subprog,

kernel/bpf/verifier.c

10826

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

10834

env->subprog_info[subprog].is_cb = true;

kernel/bpf/verifier.c

10851

env->subprog_info[subprog].is_async_cb = true;

kernel/bpf/verifier.c

10852

async_cb = push_async_cb(env, env->subprog_info[subprog].start,

kernel/bpf/verifier.c

10853

insn_idx, subprog,

kernel/bpf/verifier.c

10871

callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);

kernel/bpf/verifier.c

10875

err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,

kernel/bpf/verifier.c

10891

int err, subprog, target_insn;

kernel/bpf/verifier.c

10894

subprog = find_subprog(env, target_insn);

kernel/bpf/verifier.c

10895

if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",

kernel/bpf/verifier.c

10900

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

10903

if (subprog_is_global(env, subprog)) {

kernel/bpf/verifier.c

10904

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

10912

if (env->subprog_info[subprog].might_sleep &&

kernel/bpf/verifier.c

10923

subprog, sub_name);

kernel/bpf/verifier.c

10929

subprog, sub_name);

kernel/bpf/verifier.c

10930

if (env->subprog_info[subprog].changes_pkt_data)

kernel/bpf/verifier.c

10933

subprog_aux(env, subprog)->called = true;

kernel/bpf/verifier.c

10947

err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);

kernel/bpf/verifier.c

10954

*insn_idx = env->subprog_info[subprog].start - 1;

kernel/bpf/verifier.c

18138

struct bpf_subprog_info *subprog;

kernel/bpf/verifier.c

18140

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

18141

subprog->changes_pkt_data = true;

kernel/bpf/verifier.c

18146

struct bpf_subprog_info *subprog;

kernel/bpf/verifier.c

18148

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

18149

subprog->might_sleep = true;

kernel/bpf/verifier.c

18469

struct bpf_subprog_info *subprog,

kernel/bpf/verifier.c

18551

subprog->keep_fastcall_stack = 1;

kernel/bpf/verifier.c

18552

subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);

kernel/bpf/verifier.c

18557

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

18562

for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {

kernel/bpf/verifier.c

18565

for (i = subprog->start; i < (subprog + 1)->start; ++i) {

kernel/bpf/verifier.c

18573

for (i = subprog->start; i < (subprog + 1)->start; ++i) {

kernel/bpf/verifier.c

18577

mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);

kernel/bpf/verifier.c

18737

static struct bpf_subprog_info *subprog;

kernel/bpf/verifier.c

18742

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/verifier.c

18743

subprog_start = subprog->start;

kernel/bpf/verifier.c

18744

subprog_end = (subprog + 1)->start;

kernel/bpf/verifier.c

18807

static struct bpf_subprog_info *subprog;

kernel/bpf/verifier.c

18817

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/verifier.c

18819

jt->items[1] = subprog->exit_idx;

kernel/bpf/verifier.c

22825

int i, j, subprog_start, subprog_end = 0, len, subprog;

kernel/bpf/verifier.c

22843

subprog = find_subprog(env, i + insn->imm + 1);

kernel/bpf/verifier.c

22844

if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",

kernel/bpf/verifier.c

22850

insn->off = subprog;

kernel/bpf/verifier.c

22985

subprog = insn->off;

kernel/bpf/verifier.c

22986

insn[0].imm = (u32)(long)func[subprog]->bpf_func;

kernel/bpf/verifier.c

22987

insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;

kernel/bpf/verifier.c

22992

subprog = insn->off;

kernel/bpf/verifier.c

22993

insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);

kernel/bpf/verifier.c

23058

subprog = find_subprog(env, i + insn->off + 1);

kernel/bpf/verifier.c

23059

insn->imm = subprog;

kernel/bpf/verifier.c

24500

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

24518

if ((subprog + 1)->start == i + 1) {

kernel/bpf/verifier.c

24519

if (modified && !subprog->keep_fastcall_stack)

kernel/bpf/verifier.c

24520

subprog->stack_depth = -subprog->fastcall_stack_off;

kernel/bpf/verifier.c

24521

subprog++;

kernel/bpf/verifier.c

24572

static int do_check_common(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

24575

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

24600

subprog);

kernel/bpf/verifier.c

24601

state->first_insn_idx = env->subprog_info[subprog].start;

kernel/bpf/verifier.c

24605

if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {

kernel/bpf/verifier.c

24606

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

24611

verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);

kernel/bpf/verifier.c

24612

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

24616

if (subprog_is_exc_cb(env, subprog)) {

kernel/bpf/verifier.c

24688

if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {

kernel/bpf/verifier.c

24969

int ret = 0, subprog = -1, i;

kernel/bpf/verifier.c

25010

subprog = i;

kernel/bpf/verifier.c

25013

if (subprog == -1) {

kernel/bpf/verifier.c

25017

if (aux->func && aux->func[subprog]->aux->exception_cb) {

kernel/bpf/verifier.c

25023

conservative = aux->func_info_aux[subprog].unreliable;

kernel/bpf/verifier.c

25036

? aux->func[subprog]->aux->changes_pkt_data

kernel/bpf/verifier.c

25045

? aux->func[subprog]->aux->might_sleep

kernel/bpf/verifier.c

25210

if (subprog == 0)

kernel/bpf/verifier.c

25213

addr = (long) tgt_prog->aux->func[subprog]->bpf_func;

kernel/bpf/verifier.c

3008

int subprog, bool is_sleepable)

kernel/bpf/verifier.c

3042

subprog /* subprog number within this prog */);

kernel/bpf/verifier.c

3625

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

3677

subprog[env->subprog_cnt].start = insn_cnt;

kernel/bpf/verifier.c

3681

verbose(env, "func#%d @%d\n", i, subprog[i].start);

kernel/bpf/verifier.c

3689

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

3694

subprog_start = subprog[cur_subprog].start;

kernel/bpf/verifier.c

3695

subprog_end = subprog[cur_subprog + 1].start;

kernel/bpf/verifier.c

3702

subprog[cur_subprog].has_tail_call = true;

kernel/bpf/verifier.c

3703

subprog[cur_subprog].tail_call_reachable = true;

kernel/bpf/verifier.c

3707

subprog[cur_subprog].has_ld_abs = true;

kernel/bpf/verifier.c

3713

subprog[cur_subprog].exit_idx = i;

kernel/bpf/verifier.c

3736

subprog_end = subprog[cur_subprog + 1].start;

kernel/bpf/verifier.c

440

static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

444

return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;

kernel/bpf/verifier.c

447

static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4510

int subprog_insn_idx, subprog;

kernel/bpf/verifier.c

4513

subprog = find_subprog(env, subprog_insn_idx);

kernel/bpf/verifier.c

4514

if (subprog < 0)

kernel/bpf/verifier.c

4517

if (subprog_is_global(env, subprog)) {

kernel/bpf/verifier.c

454

info = &env->prog->aux->func_info[subprog];

kernel/bpf/verifier.c

458

static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

460

struct bpf_subprog_info *info = subprog_info(env, subprog);

kernel/bpf/verifier.c

467

static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

469

return subprog_info(env, subprog)->is_exception_cb;

kernel/bpf/verifier.c

5223

struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];

kernel/bpf/verifier.c

5227

if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)

kernel/bpf/verifier.c

5234

subprog->fastcall_stack_off = S16_MIN;

kernel/bpf/verifier.c

5238

for (i = subprog->start; i < (subprog + 1)->start; ++i) {

kernel/bpf/verifier.c

6743

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

6751

i = subprog[idx].start;

kernel/bpf/verifier.c

6753

subprog[idx].priv_stack_mode = NO_PRIV_STACK;

kernel/bpf/verifier.c

6774

if (idx && subprog[idx].has_tail_call && depth >= 256) {

kernel/bpf/verifier.c

6781

subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

6787

if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&

kernel/bpf/verifier.c

6789

subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;

kernel/bpf/verifier.c

6792

if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {

kernel/bpf/verifier.c

6807

subprog_end = subprog[idx + 1].start;

kernel/bpf/verifier.c

6816

if (subprog[idx].is_cb)

kernel/bpf/verifier.c

6819

if (subprog[ret_prog[c]].is_cb) {

kernel/bpf/verifier.c

6843

if (subprog[sidx].is_async_cb) {

kernel/bpf/verifier.c

6844

if (subprog[sidx].has_tail_call) {

kernel/bpf/verifier.c

6851

if (subprog[sidx].is_exception_cb) {

kernel/bpf/verifier.c

6859

subprog[idx].priv_stack_mode = NO_PRIV_STACK;

kernel/bpf/verifier.c

6861

if (subprog[idx].has_tail_call)

kernel/bpf/verifier.c

6879

if (subprog[ret_prog[j]].is_exception_cb) {

kernel/bpf/verifier.c

6883

subprog[ret_prog[j]].tail_call_reachable = true;

kernel/bpf/verifier.c

6885

if (subprog[0].tail_call_reachable)

kernel/bpf/verifier.c

6893

if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)

kernel/bpf/verifier.c

6894

depth -= round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

6949

int start = idx + insn->imm + 1, subprog;

kernel/bpf/verifier.c

6951

subprog = find_subprog(env, start);

kernel/bpf/verifier.c

6952

if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))

kernel/bpf/verifier.c

6954

return env->subprog_info[subprog].stack_depth;

kernel/events/core.c

10266

struct bpf_prog *subprog = prog->aux->func[i];

kernel/events/core.c

10270

(u64)(unsigned long)subprog->bpf_func,

kernel/events/core.c

10271

subprog->jited_len, unregister,

kernel/events/core.c

10272

subprog->aux->ksym.name);

tools/lib/bpf/libbpf.c

6627

static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)

tools/lib/bpf/libbpf.c

6629

int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;

tools/lib/bpf/libbpf.c

6633

if (main_prog == subprog)

tools/lib/bpf/libbpf.c

6642

if (subprog->nr_reloc)

tools/lib/bpf/libbpf.c

6643

memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,

tools/lib/bpf/libbpf.c

6644

sizeof(*relos) * subprog->nr_reloc);

tools/lib/bpf/libbpf.c

6647

relos[i].insn_idx += subprog->sub_insn_off;

tools/lib/bpf/libbpf.c

6656

static int save_subprog_offsets(struct bpf_program *main_prog, struct bpf_program *subprog)

tools/lib/bpf/libbpf.c

6667

main_prog->subprogs[cnt].sec_insn_off = subprog->sec_insn_off;

tools/lib/bpf/libbpf.c

6668

main_prog->subprogs[cnt].sub_insn_off = subprog->sub_insn_off;

tools/lib/bpf/libbpf.c

6676

struct bpf_program *subprog)

tools/lib/bpf/libbpf.c

6682

subprog->sub_insn_off = main_prog->insns_cnt;

tools/lib/bpf/libbpf.c

6684

new_cnt = main_prog->insns_cnt + subprog->insns_cnt;

tools/lib/bpf/libbpf.c

6693

memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,

tools/lib/bpf/libbpf.c

6694

subprog->insns_cnt * sizeof(*insns));

tools/lib/bpf/libbpf.c

6697

main_prog->name, subprog->insns_cnt, subprog->name);

tools/lib/bpf/libbpf.c

6700

err = append_subprog_relos(main_prog, subprog);

tools/lib/bpf/libbpf.c

6704

err = save_subprog_offsets(main_prog, subprog);

tools/lib/bpf/libbpf.c

6719

struct bpf_program *subprog;

tools/lib/bpf/libbpf.c

6778

subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);

tools/lib/bpf/libbpf.c

6779

if (!subprog) {

tools/lib/bpf/libbpf.c

6795

if (subprog->sub_insn_off == 0) {

tools/lib/bpf/libbpf.c

6796

err = bpf_object__append_subprog_code(obj, main_prog, subprog);

tools/lib/bpf/libbpf.c

6799

err = bpf_object__reloc_code(obj, main_prog, subprog);

tools/lib/bpf/libbpf.c

6814

insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;

tools/lib/bpf/libbpf.c

6817

prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);

tools/lib/bpf/libbpf.c

6907

struct bpf_program *subprog;

tools/lib/bpf/libbpf.c

6914

subprog = &obj->programs[i];

tools/lib/bpf/libbpf.c

6915

if (!prog_is_subprog(obj, subprog))

tools/lib/bpf/libbpf.c

6918

subprog->sub_insn_off = 0;

tools/lib/bpf/libbpf.c

7018

struct bpf_program *subprog = &obj->programs[j];

tools/lib/bpf/libbpf.c

7020

if (!prog_is_subprog(obj, subprog))

tools/lib/bpf/libbpf.c

7022

if (strcmp(name, subprog->name) != 0)

tools/lib/bpf/libbpf.c

7028

if (!subprog->sym_global || subprog->mark_btf_static) {

tools/lib/bpf/libbpf.c

7030

prog->name, subprog->name);

tools/lib/bpf/libbpf.c

7036

prog->name, subprog->name);

tools/lib/bpf/libbpf.c

7444

struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];

tools/lib/bpf/libbpf.c

7451

if (subprog->sub_insn_off == 0) {

tools/lib/bpf/libbpf.c

7452

err = bpf_object__append_subprog_code(obj, prog, subprog);

tools/lib/bpf/libbpf.c

7455

err = bpf_object__reloc_code(obj, prog, subprog);

tools/testing/selftests/bpf/progs/epilogue_tailcall.c

21

subprog(args);

tools/testing/selftests/bpf/progs/exceptions.c

187

return subprog(ctx);

tools/testing/selftests/bpf/progs/linked_funcs1.c

28

output_val1 = x + subprog(x);

tools/testing/selftests/bpf/progs/linked_funcs2.c

28

output_val2 = 2 * x + 2 * subprog(x);

tools/testing/selftests/bpf/progs/stream.c

206

subprog();

tools/testing/selftests/bpf/progs/test_static_linked1.c

24

var1 = subprog(rovar1) + static_var1 + static_var2;

tools/testing/selftests/bpf/progs/test_static_linked2.c

24

var2 = subprog(rovar2) + static_var1 + static_var2;