arch/alpha/include/asm/err_ev7.h

182

struct ev7_pal_environmental_subpacket env; /* Type 10-16 */

arch/alpha/include/asm/err_ev7.h

196

struct ev7_pal_environmental_subpacket *env[7]; /* Type 10-16 */

arch/alpha/kernel/err_ev7.c

104

lf_subpackets->env[ev7_lf_env_index(subpacket->type)] =

arch/alpha/kernel/err_marvel.c

28

struct ev7_pal_environmental_subpacket *env;

arch/alpha/kernel/err_marvel.c

49

env = lf_subpackets->env[ev7_lf_env_index(ev_packets[i].type)];

arch/alpha/kernel/err_marvel.c

50

if (!env)

arch/alpha/kernel/err_marvel.c

56

env->cabinet,

arch/alpha/kernel/err_marvel.c

57

env->drawer);

arch/alpha/kernel/err_marvel.c

61

env->module_type,

arch/alpha/kernel/err_marvel.c

62

env->unit_id,

arch/alpha/kernel/err_marvel.c

63

env->condition);

arch/alpha/kernel/err_marvel.c

77

if (lf_subpackets->env[i])

arch/arc/net/bpf_jit_core.c

1403

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/arm/net/bpf_jit_32.c

2151

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/arm64/net/bpf_jit_comp.c

1253

static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn *insn,

arch/arm64/net/bpf_jit_comp.c

1279

if (bpf_insn_is_indirect_target(env, ctx->prog, i))

arch/arm64/net/bpf_jit_comp.c

1978

static int build_body(struct bpf_verifier_env *env, struct jit_ctx *ctx, bool extra_pass)

arch/arm64/net/bpf_jit_comp.c

1997

ret = build_insn(env, insn, ctx, extra_pass);

arch/arm64/net/bpf_jit_comp.c

2080

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/arm64/net/bpf_jit_comp.c

2143

out_cnt = bpf_out_stack_arg_cnt(env, prog);

arch/arm64/net/bpf_jit_comp.c

2162

if (build_body(env, &ctx, extra_pass))

arch/arm64/net/bpf_jit_comp.c

2210

if (build_body(env, &ctx, extra_pass))

arch/arm64/net/bpf_jit_comp.c

2219

if (build_body(env, &ctx, extra_pass) ||

arch/loongarch/kvm/main.c

295

unsigned long env, gcfg = 0;

arch/loongarch/kvm/main.c

297

env = read_csr_gcfg();

arch/loongarch/kvm/main.c

315

if (env & CSR_GCFG_GCIP_SECURE)

arch/loongarch/kvm/main.c

317

if (env & CSR_GCFG_MATP_ROOT)

arch/loongarch/net/bpf_jit.c

2197

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/mips/alchemy/common/prom.c

67

char **env = prom_envp;

arch/mips/alchemy/common/prom.c

69

int yamon = (*env && strchr(*env, '=') == NULL);

arch/mips/alchemy/common/prom.c

71

while (*env) {

arch/mips/alchemy/common/prom.c

73

if (strcmp(envname, *env++) == 0)

arch/mips/alchemy/common/prom.c

74

return *env;

arch/mips/alchemy/common/prom.c

75

} else if (strncmp(envname, *env, i) == 0 && (*env)[i] == '=')

arch/mips/alchemy/common/prom.c

76

return *env + i + 1;

arch/mips/alchemy/common/prom.c

77

env++;

arch/mips/net/bpf_jit_comp.c

912

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/mips/sgi-ip22/ip22-gio.c

190

static int gio_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

arch/mips/sgi-ip22/ip22-gio.c

194

add_uevent_var(env, "MODALIAS=gio:%x", gio_dev->id.id);

arch/parisc/kernel/drivers.c

564

static int parisc_uevent(const struct device *dev, struct kobj_uevent_env *env)

arch/parisc/kernel/drivers.c

576

if (add_uevent_var(env, "PARISC_NAME=%s", padev->name))

arch/parisc/kernel/drivers.c

580

if (add_uevent_var(env, "MODALIAS=%s", modalias))

arch/parisc/net/bpf_jit_core.c

44

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/powerpc/include/asm/elf.h

192

unsigned long env;

arch/powerpc/include/asm/setjmp.h

12

extern int setjmp(jmp_buf env) __attribute__((returns_twice));

arch/powerpc/include/asm/setjmp.h

13

extern void longjmp(jmp_buf env, int val) __attribute__((noreturn));

arch/powerpc/net/bpf_jit_comp.c

165

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)

arch/powerpc/platforms/ps3/system-bus.c

442

static int ps3_system_bus_uevent(const struct device *_dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/ps3/system-bus.c

446

if (add_uevent_var(env, "MODALIAS=ps3:%d:%d", dev->match_id,

arch/powerpc/platforms/pseries/ibmebus.c

432

static int ibmebus_bus_modalias(const struct device *dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/pseries/ibmebus.c

434

return of_device_uevent_modalias(dev, env);

arch/powerpc/platforms/pseries/vio.c

1587

static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/pseries/vio.c

1595

add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, cp);

arch/riscv/net/bpf_jit_core.c

44

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/s390/net/bpf_jit_comp.c

2340

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)

arch/sparc/kernel/ldc.c

1612

p->env = (data_len |

arch/sparc/kernel/ldc.c

1622

p->env,

arch/sparc/kernel/ldc.c

1774

p->env,

arch/sparc/kernel/ldc.c

1816

pkt_len = p->env & LDC_LEN;

arch/sparc/kernel/ldc.c

1831

if ((first_frag == NULL && !(p->env & LDC_START)) ||

arch/sparc/kernel/ldc.c

1832

(first_frag != NULL && (p->env & LDC_START))) {

arch/sparc/kernel/ldc.c

1872

if (p->env & LDC_STOP)

arch/sparc/kernel/ldc.c

435

p->env = lp->cfg.mode;

arch/sparc/kernel/ldc.c

440

p->env, p->seqid);

arch/sparc/kernel/ldc.c

455

p->env = lp->cfg.mode;

arch/sparc/kernel/ldc.c

459

p->env, p->seqid);

arch/sparc/kernel/ldc.c

474

p->env = 0;

arch/sparc/kernel/ldc.c

479

p->env, p->seqid, p->u.r.ackid);

arch/sparc/kernel/ldc.c

61

u8 env;

arch/sparc/kernel/ldc.c

673

p->stype, p->seqid, p->env);

arch/sparc/kernel/ldc.c

677

p->env != lp->cfg.mode)

arch/sparc/kernel/ldc.c

693

p->stype, p->seqid, p->env);

arch/sparc/kernel/ldc.c

696

p->env != lp->cfg.mode)

arch/sparc/kernel/ldc.c

716

p->stype, p->seqid, p->env, p->u.r.ackid);

arch/sparc/kernel/vio.c

51

static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env)

arch/sparc/kernel/vio.c

55

add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, vio_dev->compat);

arch/sparc/net/bpf_jit_comp_64.c

1480

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/um/drivers/port_user.c

171

char *env;

arch/um/drivers/port_user.c

176

if ((env = getenv("UML_PORT_HELPER")))

arch/um/drivers/port_user.c

177

argv[2] = env;

arch/um/drivers/port_user.c

187

if (env == NULL)

arch/x86/include/asm/fpu/signal.h

21

extern void convert_from_fxsr(struct user_i387_ia32_struct *env,

arch/x86/include/asm/fpu/signal.h

24

const struct user_i387_ia32_struct *env);

arch/x86/kernel/fpu/regset.c

331

static void __convert_from_fxsr(struct user_i387_ia32_struct *env,

arch/x86/kernel/fpu/regset.c

335

struct _fpreg *to = (struct _fpreg *) &env->st_space[0];

arch/x86/kernel/fpu/regset.c

339

env->cwd = fxsave->cwd | 0xffff0000u;

arch/x86/kernel/fpu/regset.c

340

env->swd = fxsave->swd | 0xffff0000u;

arch/x86/kernel/fpu/regset.c

341

env->twd = twd_fxsr_to_i387(fxsave);

arch/x86/kernel/fpu/regset.c

344

env->fip = fxsave->rip;

arch/x86/kernel/fpu/regset.c

345

env->foo = fxsave->rdp;

arch/x86/kernel/fpu/regset.c

350

env->fcs = task_pt_regs(tsk)->cs;

arch/x86/kernel/fpu/regset.c

352

savesegment(ds, env->fos);

arch/x86/kernel/fpu/regset.c

354

env->fos = tsk->thread.ds;

arch/x86/kernel/fpu/regset.c

356

env->fos |= 0xffff0000;

arch/x86/kernel/fpu/regset.c

358

env->fip = fxsave->fip;

arch/x86/kernel/fpu/regset.c

359

env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);

arch/x86/kernel/fpu/regset.c

360

env->foo = fxsave->foo;

arch/x86/kernel/fpu/regset.c

361

env->fos = fxsave->fos;

arch/x86/kernel/fpu/regset.c

369

convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)

arch/x86/kernel/fpu/regset.c

371

__convert_from_fxsr(env, tsk, &x86_task_fpu(tsk)->fpstate->regs.fxsave);

arch/x86/kernel/fpu/regset.c

375

const struct user_i387_ia32_struct *env)

arch/x86/kernel/fpu/regset.c

378

struct _fpreg *from = (struct _fpreg *) &env->st_space[0];

arch/x86/kernel/fpu/regset.c

382

fxsave->cwd = env->cwd;

arch/x86/kernel/fpu/regset.c

383

fxsave->swd = env->swd;

arch/x86/kernel/fpu/regset.c

384

fxsave->twd = twd_i387_to_fxsr(env->twd);

arch/x86/kernel/fpu/regset.c

385

fxsave->fop = (u16) ((u32) env->fcs >> 16);

arch/x86/kernel/fpu/regset.c

387

fxsave->rip = env->fip;

arch/x86/kernel/fpu/regset.c

388

fxsave->rdp = env->foo;

arch/x86/kernel/fpu/regset.c

391

fxsave->fip = env->fip;

arch/x86/kernel/fpu/regset.c

392

fxsave->fcs = (env->fcs & 0xffff);

arch/x86/kernel/fpu/regset.c

393

fxsave->foo = env->foo;

arch/x86/kernel/fpu/regset.c

394

fxsave->fos = env->fos;

arch/x86/kernel/fpu/regset.c

405

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/regset.c

428

__convert_from_fxsr(&env, target, fx);

arch/x86/kernel/fpu/regset.c

429

return membuf_write(&to, &env, sizeof(env));

arch/x86/kernel/fpu/regset.c

437

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/regset.c

447

ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);

arch/x86/kernel/fpu/regset.c

454

convert_to_fxsr(&fpu->fpstate->regs.fxsave, &env);

arch/x86/kernel/fpu/regset.c

456

memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env));

arch/x86/kernel/fpu/signal.c

332

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/signal.c

359

if (__copy_from_user(&env, buf, sizeof(env)))

arch/x86/kernel/fpu/signal.c

410

convert_to_fxsr(&fpregs->fxsave, &env);

arch/x86/kernel/fpu/signal.c

73

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/signal.c

81

convert_from_fxsr(&env, tsk);

arch/x86/kernel/fpu/signal.c

83

if (__copy_to_user(buf, &env, sizeof(env)) ||

arch/x86/net/bpf_jit_comp.c

1681

static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,

arch/x86/net/bpf_jit_comp.c

1703

out_stack_arg_cnt = bpf_out_stack_arg_cnt(env, bpf_prog);

arch/x86/net/bpf_jit_comp.c

1830

if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))

arch/x86/net/bpf_jit_comp.c

3851

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

arch/x86/net/bpf_jit_comp.c

3929

proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen,

arch/x86/net/bpf_jit_comp32.c

2521

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

block/genhd.c

1316

static int block_uevent(const struct device *dev, struct kobj_uevent_env *env)

block/genhd.c

1320

return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);

block/partitions/core.c

254

static int part_uevent(const struct device *dev, struct kobj_uevent_env *env)

block/partitions/core.c

258

add_uevent_var(env, "PARTN=%u", bdev_partno(part));

block/partitions/core.c

260

add_uevent_var(env, "PARTNAME=%s", part->bd_meta_info->volname);

block/partitions/core.c

262

add_uevent_var(env, "PARTUUID=%s", part->bd_meta_info->uuid);

drivers/acpi/bus.c

1177

static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/acpi/bus.c

1179

return __acpi_device_uevent_modalias(to_acpi_device(dev), env);

drivers/acpi/device_sysfs.c

241

struct kobj_uevent_env *env)

drivers/acpi/device_sysfs.c

251

if (add_uevent_var(env, "MODALIAS="))

drivers/acpi/device_sysfs.c

255

len = create_of_modalias(adev, &env->buf[env->buflen - 1],

drivers/acpi/device_sysfs.c

256

sizeof(env->buf) - env->buflen);

drivers/acpi/device_sysfs.c

258

len = create_pnp_modalias(adev, &env->buf[env->buflen - 1],

drivers/acpi/device_sysfs.c

259

sizeof(env->buf) - env->buflen);

drivers/acpi/device_sysfs.c

263

env->buflen += len;

drivers/acpi/device_sysfs.c

278

int acpi_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/acpi/device_sysfs.c

280

return __acpi_device_uevent_modalias(acpi_companion_match(dev), env);

drivers/acpi/internal.h

137

struct kobj_uevent_env *env);

drivers/amba/bus.c

213

static int amba_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/amba/bus.c

218

retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid);

drivers/amba/bus.c

222

retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid);

drivers/base/auxiliary.c

199

static int auxiliary_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/auxiliary.c

206

return add_uevent_var(env, "MODALIAS=%s%.*s", AUXILIARY_MODULE_PREFIX,

drivers/base/core.c

2708

static void dev_driver_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/core.c

2716

add_uevent_var(env, "DRIVER=%s", drv->name);

drivers/base/core.c

2723

static int dev_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

drivers/base/core.c

2736

add_uevent_var(env, "MAJOR=%u", MAJOR(dev->devt));

drivers/base/core.c

2737

add_uevent_var(env, "MINOR=%u", MINOR(dev->devt));

drivers/base/core.c

2740

add_uevent_var(env, "DEVNAME=%s", name);

drivers/base/core.c

2742

add_uevent_var(env, "DEVMODE=%#o", mode & 0777);

drivers/base/core.c

2744

add_uevent_var(env, "DEVUID=%u", from_kuid(&init_user_ns, uid));

drivers/base/core.c

2746

add_uevent_var(env, "DEVGID=%u", from_kgid(&init_user_ns, gid));

drivers/base/core.c

2752

add_uevent_var(env, "DEVTYPE=%s", dev->type->name);

drivers/base/core.c

2755

dev_driver_uevent(dev, env);

drivers/base/core.c

2758

of_device_uevent(dev, env);

drivers/base/core.c

2762

retval = dev->bus->uevent(dev, env);

drivers/base/core.c

2770

retval = dev->class->dev_uevent(dev, env);

drivers/base/core.c

2779

retval = dev->type->uevent(dev, env);

drivers/base/core.c

2800

struct kobj_uevent_env *env = NULL;

drivers/base/core.c

2821

env = kzalloc_obj(struct kobj_uevent_env);

drivers/base/core.c

2822

if (!env)

drivers/base/core.c

2826

retval = kset->uevent_ops->uevent(&dev->kobj, env);

drivers/base/core.c

2831

for (i = 0; i < env->envp_idx; i++)

drivers/base/core.c

2832

len += sysfs_emit_at(buf, len, "%s\n", env->envp[i]);

drivers/base/core.c

2834

kfree(env);

drivers/base/cpu.c

382

static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/cpu.c

387

add_uevent_var(env, "MODALIAS=%s", buf);

drivers/base/firmware_loader/sysfs.c

70

static int do_firmware_uevent(const struct fw_sysfs *fw_sysfs, struct kobj_uevent_env *env)

drivers/base/firmware_loader/sysfs.c

72

if (add_uevent_var(env, "FIRMWARE=%s", fw_sysfs->fw_priv->fw_name))

drivers/base/firmware_loader/sysfs.c

74

if (add_uevent_var(env, "TIMEOUT=%i", __firmware_loading_timeout()))

drivers/base/firmware_loader/sysfs.c

76

if (add_uevent_var(env, "ASYNC=%d", fw_sysfs->nowait))

drivers/base/firmware_loader/sysfs.c

82

static int firmware_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/firmware_loader/sysfs.c

89

err = do_firmware_uevent(fw_sysfs, env);

drivers/base/platform.c

1395

static int platform_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/platform.c

1401

rc = of_device_uevent_modalias(dev, env);

drivers/base/platform.c

1405

rc = acpi_device_uevent_modalias(dev, env);

drivers/base/platform.c

1409

add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX, pdev->name);

drivers/bcma/main.c

32

static int bcma_device_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/bcma/main.c

637

static int bcma_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bcma/main.c

641

return add_uevent_var(env,

drivers/bus/fsl-mc/fsl-mc-bus.c

139

static int fsl_mc_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/fsl-mc/fsl-mc-bus.c

143

if (add_uevent_var(env, "MODALIAS=fsl-mc:v%08Xd%s",

drivers/bus/mhi/ep/main.c

1673

static int mhi_ep_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mhi/ep/main.c

1677

return add_uevent_var(env, "MODALIAS=" MHI_EP_DEVICE_MODALIAS_FMT,

drivers/bus/mhi/host/init.c

1419

static int mhi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mhi/host/init.c

1423

return add_uevent_var(env, "MODALIAS=" MHI_DEVICE_MODALIAS_FMT,

drivers/bus/mips_cdmm.c

70

static int mips_cdmm_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mips_cdmm.c

75

retval = add_uevent_var(env, "CDMM_CPU=%u", cdev->cpu);

drivers/bus/mips_cdmm.c

79

retval = add_uevent_var(env, "CDMM_TYPE=0x%02x", cdev->type);

drivers/bus/mips_cdmm.c

83

retval = add_uevent_var(env, "CDMM_REV=%u", cdev->rev);

drivers/bus/mips_cdmm.c

87

retval = add_uevent_var(env, "MODALIAS=mipscdmm:t%02X", cdev->type);

drivers/bus/sunxi-rsb.c

175

static int sunxi_rsb_device_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/sunxi-rsb.c

177

return of_device_uevent_modalias(dev, env);

drivers/cxl/core/port.c

2249

static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/cxl/core/port.c

2251

return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,

drivers/dax/bus.c

34

static int dax_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/dax/bus.c

40

return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);

drivers/dma/idxd/bus.c

70

static int idxd_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/dma/idxd/bus.c

72

return add_uevent_var(env, "MODALIAS=" IDXD_DEVICES_MODALIAS_FMT, 0);

drivers/eisa/eisa-bus.c

130

static int eisa_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/eisa/eisa-bus.c

134

add_uevent_var(env, "MODALIAS=" EISA_DEVICE_MODALIAS_FMT, edev->id.sig);

drivers/firewire/core-device.c

241

static int fw_unit_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firewire/core-device.c

248

if (add_uevent_var(env, "MODALIAS=%s", modalias))

drivers/firmware/arm_ffa/bus.c

69

static int ffa_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/arm_ffa/bus.c

73

return add_uevent_var(env, "MODALIAS=" FFA_UEVENT_MODALIAS_FMT,

drivers/firmware/arm_scmi/bus.c

280

static int scmi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/arm_scmi/bus.c

284

return add_uevent_var(env, "MODALIAS=" SCMI_UEVENT_MODALIAS_FMT,

drivers/firmware/dmi-id.c

159

static int dmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/dmi-id.c

163

if (add_uevent_var(env, "MODALIAS="))

drivers/firmware/dmi-id.c

165

len = get_modalias(&env->buf[env->buflen - 1],

drivers/firmware/dmi-id.c

166

sizeof(env->buf) - env->buflen);

drivers/firmware/dmi-id.c

167

if (len >= (sizeof(env->buf) - env->buflen))

drivers/firmware/dmi-id.c

169

env->buflen += len;

drivers/firmware/google/coreboot_table.c

76

static int coreboot_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/google/coreboot_table.c

81

return add_uevent_var(env, "MODALIAS=coreboot:t%08X", tag);

drivers/fpga/dfl.c

285

static int dfl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/fpga/dfl.c

289

return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X",

drivers/gnss/core.c

367

static int gnss_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/gnss/core.c

372

ret = add_uevent_var(env, "GNSS_TYPE=%s", gnss_type_name(gdev));

drivers/gpu/drm/display/drm_dp_aux_bus.c

165

static int dp_aux_ep_dev_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/gpu/drm/display/drm_dp_aux_bus.c

167

return of_device_uevent_modalias(dev, env);

drivers/gpu/drm/drm_mipi_dsi.c

69

static int mipi_dsi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/gpu/drm/drm_mipi_dsi.c

74

err = of_device_uevent_modalias(dev, env);

drivers/gpu/drm/drm_mipi_dsi.c

78

add_uevent_var(env, "MODALIAS=%s%s", MIPI_DSI_MODULE_PREFIX,

drivers/gpu/drm/i915/gvt/handlers.c

1526

char *env[3] = {NULL, NULL, NULL};

drivers/gpu/drm/i915/gvt/handlers.c

1531

env[0] = display_ready_str;

drivers/gpu/drm/i915/gvt/handlers.c

1534

env[1] = vmid_str;

drivers/gpu/drm/i915/gvt/handlers.c

1536

return kobject_uevent_env(kobj, KOBJ_ADD, env);

drivers/gpu/host1x/bus.c

342

struct kobj_uevent_env *env)

drivers/gpu/host1x/bus.c

344

of_device_uevent(dev->parent, env);

drivers/greybus/core.c

111

static int greybus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/greybus/core.c

147

if (add_uevent_var(env, "BUS=%u", hd->bus_id))

drivers/greybus/core.c

151

if (add_uevent_var(env, "MODULE=%u", module->module_id))

drivers/greybus/core.c

156

if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))

drivers/greybus/core.c

158

if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",

drivers/greybus/core.c

169

if (add_uevent_var(env, "BUNDLE=%u", bundle->id))

drivers/greybus/core.c

171

if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))

drivers/hid/hid-core.c

2924

static int hid_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hid/hid-core.c

2928

if (add_uevent_var(env, "HID_ID=%04X:%08X:%08X",

drivers/hid/hid-core.c

2932

if (add_uevent_var(env, "HID_NAME=%s", hdev->name))

drivers/hid/hid-core.c

2935

if (add_uevent_var(env, "HID_PHYS=%s", hdev->phys))

drivers/hid/hid-core.c

2938

if (add_uevent_var(env, "HID_UNIQ=%s", hdev->uniq))

drivers/hid/hid-core.c

2941

if (add_uevent_var(env, "MODALIAS=hid:b%04Xg%04Xv%08Xp%08X",

drivers/hid/hid-core.c

2945

if (add_uevent_var(env, "HID_FIRMWARE_VERSION=0x%04llX",

drivers/hid/intel-ish-hid/ishtp/bus.c

372

static int ishtp_cl_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hid/intel-ish-hid/ishtp/bus.c

374

if (add_uevent_var(env, "MODALIAS=" ISHTP_MODULE_PREFIX "%s", dev_name(dev)))

drivers/hsi/hsi_core.c

33

static int hsi_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hsi/hsi_core.c

35

add_uevent_var(env, "MODALIAS=hsi:%s", dev_name(dev));

drivers/hv/vmbus_drv.c

642

static int vmbus_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/hv/vmbus_drv.c

647

return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type);

drivers/hwmon/ultra45_env.c

102

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

121

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

144

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

167

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

188

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

204

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

257

struct env *p = devm_kzalloc(&op->dev, sizeof(*p), GFP_KERNEL);

drivers/hwmon/ultra45_env.c

296

struct env *p = platform_get_drvdata(op);

drivers/hwmon/ultra45_env.c

69

static u8 env_read(struct env *p, u8 ireg)

drivers/hwmon/ultra45_env.c

81

static void env_write(struct env *p, u8 ireg, u8 val)

drivers/i2c/i2c-core-base.c

163

static int i2c_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/i2c/i2c-core-base.c

168

rc = of_device_uevent_modalias(dev, env);

drivers/i2c/i2c-core-base.c

172

rc = acpi_device_uevent_modalias(dev, env);

drivers/i2c/i2c-core-base.c

176

return add_uevent_var(env, "MODALIAS=%s%s", I2C_MODULE_PREFIX, client->name);

drivers/i3c/master.c

314

static int i3c_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/i3c/master.c

327

return add_uevent_var(env, "MODALIAS=i3c:dcr%02Xmanuf%04X",

drivers/i3c/master.c

330

return add_uevent_var(env,

drivers/iio/adc/envelope-detector.c

100

synchronize_irq(env->comp_irq);

drivers/iio/adc/envelope-detector.c

103

spin_lock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

104

comp = env->comp;

drivers/iio/adc/envelope-detector.c

105

env->comp = 0;

drivers/iio/adc/envelope-detector.c

106

spin_unlock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

109

enable_irq(env->comp_irq);

drivers/iio/adc/envelope-detector.c

116

struct envelope *env = ctx;

drivers/iio/adc/envelope-detector.c

118

spin_lock(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

119

env->comp = 1;

drivers/iio/adc/envelope-detector.c

120

disable_irq_nosync(env->comp_irq);

drivers/iio/adc/envelope-detector.c

121

spin_unlock(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

126

static void envelope_detector_setup_compare(struct envelope *env)

drivers/iio/adc/envelope-detector.c

140

env->level = (env->high + env->low + !env->invert) / 2;

drivers/iio/adc/envelope-detector.c

142

if (env->high == env->low + 1) {

drivers/iio/adc/envelope-detector.c

143

complete(&env->done);

drivers/iio/adc/envelope-detector.c

148

ret = iio_write_channel_raw(env->dac, env->invert ? 0 : env->dac_max);

drivers/iio/adc/envelope-detector.c

153

envelope_detector_comp_latch(env);

drivers/iio/adc/envelope-detector.c

156

ret = iio_write_channel_raw(env->dac, env->level);

drivers/iio/adc/envelope-detector.c

161

schedule_delayed_work(&env->comp_timeout,

drivers/iio/adc/envelope-detector.c

162

msecs_to_jiffies(env->comp_interval));

drivers/iio/adc/envelope-detector.c

166

env->level = ret;

drivers/iio/adc/envelope-detector.c

167

complete(&env->done);

drivers/iio/adc/envelope-detector.c

172

struct envelope *env = container_of(work, struct envelope,

drivers/iio/adc/envelope-detector.c

176

if (!envelope_detector_comp_latch(env) ^ !env->invert)

drivers/iio/adc/envelope-detector.c

177

env->low = env->level;

drivers/iio/adc/envelope-detector.c

179

env->high = env->level;

drivers/iio/adc/envelope-detector.c

182

envelope_detector_setup_compare(env);

drivers/iio/adc/envelope-detector.c

189

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

204

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

205

env->high = env->dac_max + env->invert;

drivers/iio/adc/envelope-detector.c

206

env->low = -1 + env->invert;

drivers/iio/adc/envelope-detector.c

207

envelope_detector_setup_compare(env);

drivers/iio/adc/envelope-detector.c

208

wait_for_completion(&env->done);

drivers/iio/adc/envelope-detector.c

209

if (env->level < 0) {

drivers/iio/adc/envelope-detector.c

210

ret = env->level;

drivers/iio/adc/envelope-detector.c

213

*val = env->invert ? env->dac_max - env->level : env->level;

drivers/iio/adc/envelope-detector.c

214

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

219

return iio_read_channel_scale(env->dac, val, val2);

drivers/iio/adc/envelope-detector.c

225

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

233

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

235

return sprintf(buf, "%u\n", env->invert);

drivers/iio/adc/envelope-detector.c

243

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

254

trigger = invert ? env->comp_irq_trigger_inv : env->comp_irq_trigger;

drivers/iio/adc/envelope-detector.c

256

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

257

if (invert != env->invert)

drivers/iio/adc/envelope-detector.c

258

ret = irq_set_irq_type(env->comp_irq, trigger);

drivers/iio/adc/envelope-detector.c

260

env->invert = invert;

drivers/iio/adc/envelope-detector.c

263

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

273

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

275

return sprintf(buf, "%u\n", env->comp_interval);

drivers/iio/adc/envelope-detector.c

283

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

293

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

294

env->comp_interval = interval;

drivers/iio/adc/envelope-detector.c

295

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

326

struct envelope *env;

drivers/iio/adc/envelope-detector.c

330

indio_dev = devm_iio_device_alloc(dev, sizeof(*env));

drivers/iio/adc/envelope-detector.c

335

env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

336

env->comp_interval = 50; /* some sensible default? */

drivers/iio/adc/envelope-detector.c

338

spin_lock_init(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

339

mutex_init(&env->read_lock);

drivers/iio/adc/envelope-detector.c

340

init_completion(&env->done);

drivers/iio/adc/envelope-detector.c

341

INIT_DELAYED_WORK(&env->comp_timeout, envelope_detector_timeout);

drivers/iio/adc/envelope-detector.c

348

env->dac = devm_iio_channel_get(dev, "dac");

drivers/iio/adc/envelope-detector.c

349

if (IS_ERR(env->dac))

drivers/iio/adc/envelope-detector.c

350

return dev_err_probe(dev, PTR_ERR(env->dac),

drivers/iio/adc/envelope-detector.c

353

env->comp_irq = platform_get_irq_byname(pdev, "comp");

drivers/iio/adc/envelope-detector.c

354

if (env->comp_irq < 0)

drivers/iio/adc/envelope-detector.c

355

return env->comp_irq;

drivers/iio/adc/envelope-detector.c

357

ret = devm_request_irq(dev, env->comp_irq, envelope_detector_comp_isr,

drivers/iio/adc/envelope-detector.c

358

0, "envelope-detector", env);

drivers/iio/adc/envelope-detector.c

362

env->comp_irq_trigger = irq_get_trigger_type(env->comp_irq);

drivers/iio/adc/envelope-detector.c

363

if (env->comp_irq_trigger & IRQF_TRIGGER_RISING)

drivers/iio/adc/envelope-detector.c

364

env->comp_irq_trigger_inv |= IRQF_TRIGGER_FALLING;

drivers/iio/adc/envelope-detector.c

365

if (env->comp_irq_trigger & IRQF_TRIGGER_FALLING)

drivers/iio/adc/envelope-detector.c

366

env->comp_irq_trigger_inv |= IRQF_TRIGGER_RISING;

drivers/iio/adc/envelope-detector.c

367

if (env->comp_irq_trigger & IRQF_TRIGGER_HIGH)

drivers/iio/adc/envelope-detector.c

368

env->comp_irq_trigger_inv |= IRQF_TRIGGER_LOW;

drivers/iio/adc/envelope-detector.c

369

if (env->comp_irq_trigger & IRQF_TRIGGER_LOW)

drivers/iio/adc/envelope-detector.c

370

env->comp_irq_trigger_inv |= IRQF_TRIGGER_HIGH;

drivers/iio/adc/envelope-detector.c

372

ret = iio_get_channel_type(env->dac, &type);

drivers/iio/adc/envelope-detector.c

381

ret = iio_read_max_channel_raw(env->dac, &env->dac_max);

drivers/iio/adc/envelope-detector.c

78

static int envelope_detector_comp_latch(struct envelope *env)

drivers/iio/adc/envelope-detector.c

82

spin_lock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

83

comp = env->comp;

drivers/iio/adc/envelope-detector.c

84

env->comp = 0;

drivers/iio/adc/envelope-detector.c

85

spin_unlock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

97

enable_irq(env->comp_irq);

drivers/infiniband/core/device.c

501

struct kobj_uevent_env *env)

drivers/infiniband/core/device.c

503

if (add_uevent_var(env, "NAME=%s", dev_name(device)))

drivers/input/input.c

1590

static int input_add_uevent_bm_var(struct kobj_uevent_env *env,

drivers/input/input.c

1595

if (add_uevent_var(env, "%s", name))

drivers/input/input.c

1598

len = input_print_bitmap(&env->buf[env->buflen - 1],

drivers/input/input.c

1599

sizeof(env->buf) - env->buflen,

drivers/input/input.c

1601

if (len >= (sizeof(env->buf) - env->buflen))

drivers/input/input.c

1604

env->buflen += len;

drivers/input/input.c

1625

static int input_add_uevent_modalias_var(struct kobj_uevent_env *env,

drivers/input/input.c

1630

if (add_uevent_var(env, "MODALIAS="))

drivers/input/input.c

1633

len = input_print_modalias(&env->buf[env->buflen - 1],

drivers/input/input.c

1634

(int)sizeof(env->buf) - env->buflen -

drivers/input/input.c

1637

if (len >= ((int)sizeof(env->buf) - env->buflen -

drivers/input/input.c

1641

env->buflen += len;

drivers/input/input.c

1647

int err = add_uevent_var(env, fmt, val); \

drivers/input/input.c

1654

int err = input_add_uevent_bm_var(env, name, bm, max); \

drivers/input/input.c

1661

int err = input_add_uevent_modalias_var(env, dev); \

drivers/input/input.c

1666

static int input_dev_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/input/serio/serio.c

865

int err = add_uevent_var(env, fmt, val); \

drivers/input/serio/serio.c

870

static int serio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ipack/ipack.c

79

static int ipack_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ipack/ipack.c

88

if (add_uevent_var(env,

drivers/macintosh/macio_asic.c

132

static int macio_device_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/macintosh/macio_asic.c

134

return of_device_uevent_modalias(dev, env);

drivers/mcb/mcb-core.c

44

static int mcb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mcb/mcb-core.c

49

ret = add_uevent_var(env, "MODALIAS=mcb:16z%03d", mdev->id);

drivers/md/bcache/super.c

1055

char *env[] = {

drivers/md/bcache/super.c

1093

kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);

drivers/md/bcache/super.c

1110

kfree(env[1]);

drivers/md/bcache/super.c

1111

kfree(env[2]);

drivers/md/bcache/sysfs.c

296

struct kobj_uevent_env *env;

drivers/md/bcache/sysfs.c

418

env = kzalloc_obj(struct kobj_uevent_env);

drivers/md/bcache/sysfs.c

419

if (!env)

drivers/md/bcache/sysfs.c

421

add_uevent_var(env, "DRIVER=bcache");

drivers/md/bcache/sysfs.c

422

add_uevent_var(env, "CACHED_UUID=%pU", dc->sb.uuid);

drivers/md/bcache/sysfs.c

423

add_uevent_var(env, "CACHED_LABEL=%s", buf);

drivers/md/bcache/sysfs.c

426

env->envp);

drivers/md/bcache/sysfs.c

427

kfree(env);

drivers/media/dvb-core/dvbdev.c

1066

static int dvb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/media/dvb-core/dvbdev.c

1070

add_uevent_var(env, "DVB_ADAPTER_NUM=%d", dvbdev->adapter->num);

drivers/media/dvb-core/dvbdev.c

1071

add_uevent_var(env, "DVB_DEVICE_TYPE=%s", dnames[dvbdev->type]);

drivers/media/dvb-core/dvbdev.c

1072

add_uevent_var(env, "DVB_DEVICE_NUM=%d", dvbdev->id);

drivers/media/rc/rc-main.c

1618

static int rc_dev_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/media/rc/rc-main.c

1628

ret = add_uevent_var(env, "NAME=%s", dev->rc_map.name);

drivers/media/rc/rc-main.c

1630

ret = add_uevent_var(env, "DRV_NAME=%s", dev->driver_name);

drivers/media/rc/rc-main.c

1632

ret = add_uevent_var(env, "DEV_NAME=%s", dev->device_name);

drivers/memstick/core/memstick.c

59

static int memstick_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/memstick/core/memstick.c

64

if (add_uevent_var(env, "MEMSTICK_TYPE=%02X", card->id.type))

drivers/memstick/core/memstick.c

67

if (add_uevent_var(env, "MEMSTICK_CATEGORY=%02X", card->id.category))

drivers/memstick/core/memstick.c

70

if (add_uevent_var(env, "MEMSTICK_CLASS=%02X", card->id.class))

drivers/misc/mei/bus.c

1282

static int mei_cl_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/misc/mei/bus.c

1288

if (add_uevent_var(env, "MEI_CL_VERSION=%d", version))

drivers/misc/mei/bus.c

1291

if (add_uevent_var(env, "MEI_CL_UUID=%pUl", uuid))

drivers/misc/mei/bus.c

1294

if (add_uevent_var(env, "MEI_CL_NAME=%s", cldev->name))

drivers/misc/mei/bus.c

1297

if (add_uevent_var(env, "MODALIAS=mei:%s:%pUl:%02X:",

drivers/misc/tifm_core.c

58

static int tifm_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/misc/tifm_core.c

62

if (add_uevent_var(env, "TIFM_CARD_TYPE=%s", tifm_media_type_name(sock->type, 1)))

drivers/mmc/core/bus.c

101

retval = add_uevent_var(env, "SDIO_INFO%u=%s", i+1, card->info[i]);

drivers/mmc/core/bus.c

114

retval = add_uevent_var(env, "MMC_NAME=%s", mmc_card_name(card));

drivers/mmc/core/bus.c

122

retval = add_uevent_var(env, "MODALIAS=mmc:block");

drivers/mmc/core/bus.c

59

mmc_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mmc/core/bus.c

84

retval = add_uevent_var(env, "MMC_TYPE=%s", type);

drivers/mmc/core/bus.c

90

retval = add_uevent_var(env, "SDIO_ID=%04X:%04X",

drivers/mmc/core/bus.c

95

retval = add_uevent_var(env, "SDIO_REVISION=%u.%u",

drivers/mmc/core/sdio_bus.c

123

sdio_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mmc/core/sdio_bus.c

128

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

132

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

136

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

141

if (add_uevent_var(env, "SDIO_INFO%u=%s", i+1, func->info[i]))

drivers/mmc/core/sdio_bus.c

145

if (add_uevent_var(env,

drivers/net/ethernet/mellanox/mlxsw/core.c

2195

err = mlxsw_env_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

2210

mlxsw_env_fini(mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

2292

mlxsw_env_fini(mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

3211

return mlxsw_core->env;

drivers/net/ethernet/mellanox/mlxsw/core.c

98

struct mlxsw_env *env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1240

static int mlxsw_env_line_cards_alloc(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1245

for (i = 0; i < env->num_of_slots; i++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1246

env->line_cards[i] = kzalloc_flex(*env->line_cards[i],

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1248

env->max_module_count);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1249

if (!env->line_cards[i])

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1255

for (j = 0; j < env->max_module_count; j++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1256

module_info = &env->line_cards[i]->module_info[j];

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1266

kfree(env->line_cards[i]);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1270

static void mlxsw_env_line_cards_free(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1272

int i = env->num_of_slots;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1275

kfree(env->line_cards[i]);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1326

struct mlxsw_env *env,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1331

for (i = 0; i < env->line_cards[slot_index]->module_count; i++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1337

module_info = &env->line_cards[slot_index]->module_info[i];

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1343

dev_err(env->bus_info->dev, "%s\n", extack._msg);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1438

struct mlxsw_env *env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1455

env = kzalloc_flex(*env, line_cards, num_of_slots + 1);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1456

if (!env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1459

env->core = mlxsw_core;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1460

env->bus_info = bus_info;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1461

env->num_of_slots = num_of_slots + 1;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1462

env->max_module_count = max_module_count;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1463

err = mlxsw_env_line_cards_alloc(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1467

mutex_init(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1468

*p_env = env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1470

err = mlxsw_linecards_event_ops_register(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1471

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1486

env->line_cards[0]->module_count = num_of_slots ? 0 : module_count;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1491

err = mlxsw_env_module_event_enable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1499

mlxsw_env_max_module_eeprom_len_query(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1500

env->line_cards[0]->active = true;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1505

mlxsw_env_module_event_disable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1507

mlxsw_env_module_plug_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1509

mlxsw_env_temp_warn_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1511

mlxsw_linecards_event_ops_unregister(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1512

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1514

mutex_destroy(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1515

mlxsw_env_line_cards_free(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1517

kfree(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1521

void mlxsw_env_fini(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1523

env->line_cards[0]->active = false;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1524

mlxsw_env_module_event_disable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1525

mlxsw_env_module_plug_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1528

mlxsw_env_temp_warn_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1529

mlxsw_linecards_event_ops_unregister(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1530

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1531

mutex_destroy(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1532

mlxsw_env_line_cards_free(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1533

kfree(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.h

72

void mlxsw_env_fini(struct mlxsw_env *env);

drivers/net/ethernet/netronome/nfp/bpf/main.h

566

int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx,

drivers/net/ethernet/netronome/nfp/bpf/main.h

568

int nfp_bpf_finalize(struct bpf_verifier_env *env);

drivers/net/ethernet/netronome/nfp/bpf/main.h

570

int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

drivers/net/ethernet/netronome/nfp/bpf/main.h

572

int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

113

pr_vlog(env, "value at offset %d/%d may be non-zero, bpf_map_update_elem() is required to initialize atomic counters to zero to avoid offload endian issues\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

124

nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

131

pr_vlog(env, "%s: unsupported ptr type %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

136

pr_vlog(env, "%s: variable pointer\n", fname);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

142

pr_vlog(env, "%s: unaligned stack pointer %lld\n", fname, -off);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

157

nfp_bpf_map_call_ok(const char *fname, struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

16

#define pr_vlog(env, fmt, ...) \

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

162

pr_vlog(env, "%s: not supported by FW\n", fname);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

17

bpf_verifier_log_write(env, "[nfp] " fmt, ##__VA_ARGS__)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

171

struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

174

const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

175

const struct bpf_reg_state *reg2 = cur_regs(env) + BPF_REG_2;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

176

const struct bpf_reg_state *reg3 = cur_regs(env) + BPF_REG_3;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

183

pr_vlog(env, "adjust_head not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

187

pr_vlog(env, "adjust_head: FW requires shifting metadata, not supported by the driver\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

196

pr_vlog(env, "adjust_tail not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

202

if (!nfp_bpf_map_call_ok("map_lookup", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

204

!nfp_bpf_stack_arg_ok("map_lookup", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

210

if (!nfp_bpf_map_call_ok("map_update", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

212

!nfp_bpf_stack_arg_ok("map_update", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

214

!nfp_bpf_stack_arg_ok("map_update", env, reg3, NULL) ||

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

215

!nfp_bpf_map_update_value_ok(env))

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

220

if (!nfp_bpf_map_call_ok("map_delete", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

222

!nfp_bpf_stack_arg_ok("map_delete", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

230

pr_vlog(env, "bpf_get_prandom_u32(): FW doesn't support random number generation\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

240

pr_vlog(env, "event_output: not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

253

pr_vlog(env, "event_output: must use BPF_F_CURRENT_CPU, var_off: %s\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

261

reg1 = cur_regs(env) + BPF_REG_4;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

267

pr_vlog(env, "event_output: unsupported ptr type: %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

273

!nfp_bpf_stack_arg_ok("event_output", env, reg1, NULL))

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

286

pr_vlog(env, "warning: return codes and behavior of event_output helper differ for offload!\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

292

pr_vlog(env, "event_output: ptr type changed: %d %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

299

pr_vlog(env, "unsupported function id: %d\n", func_id);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

312

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

314

const struct bpf_reg_state *reg0 = cur_regs(env) + BPF_REG_0;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

324

pr_vlog(env, "unsupported exit state: %d, var_off: %s\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

334

pr_vlog(env, "unsupported exit state: %d, imm: %llx\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

346

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

350

if (reg->frameno != env->cur_state->curframe)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

354

pr_vlog(env, "variable ptr stack access\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

372

pr_vlog(env, "stack access changed location was:%d is:%d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

392

nfp_bpf_map_mark_used_one(struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

398

pr_vlog(env, "map value use type conflict %s vs %s off: %u\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

406

pr_vlog(env, "atomic counter in map value may already be initialized to non-zero value off: %u\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

417

nfp_bpf_map_mark_used(struct bpf_verifier_env *env, struct nfp_insn_meta *meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

427

pr_vlog(env, "map value offset is variable\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

437

pr_vlog(env, "map value access out-of-bounds\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

442

err = nfp_bpf_map_mark_used_one(env, nfp_map, off + i, use);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

452

struct bpf_verifier_env *env, u8 reg_no)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

454

const struct bpf_reg_state *reg = cur_regs(env) + reg_no;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

461

pr_vlog(env, "unsupported ptr type: %d\n", reg->type);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

466

err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

473

err = nfp_bpf_map_mark_used(env, meta, reg,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

479

pr_vlog(env, "map writes not supported\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

483

err = nfp_bpf_map_mark_used(env, meta, reg,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

491

pr_vlog(env, "ptr type changed for instruction %d -> %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

503

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

505

const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

514

pr_vlog(env, "queue selection not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

518

pr_vlog(env, "unsupported store to context field\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

522

return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

527

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

529

const struct bpf_reg_state *sreg = cur_regs(env) + meta->insn.src_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

530

const struct bpf_reg_state *dreg = cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

533

pr_vlog(env, "atomic op not implemented: %d\n", meta->insn.imm);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

538

pr_vlog(env, "atomic add not to a map value pointer: %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

543

pr_vlog(env, "atomic add not of a scalar: %d\n", sreg->type);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

552

return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

557

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

560

cur_regs(env) + meta->insn.src_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

562

cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

583

pr_vlog(env, "multiplier is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

587

pr_vlog(env, "multiplicand is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

592

pr_vlog(env, "sign extended multiplicand won't be within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

609

pr_vlog(env, "dividend is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

614

pr_vlog(env, "divisor is not constant\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

618

pr_vlog(env, "divisor is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

623

pr_vlog(env, "divide by negative constant is not supported\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

631

int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

634

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

641

pr_vlog(env, "instruction %#02x not supported\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

648

pr_vlog(env, "program uses extended registers - jit hardening?\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

653

return nfp_bpf_check_helper_call(nfp_prog, env, meta);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

655

return nfp_bpf_check_exit(nfp_prog, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

658

return nfp_bpf_check_ptr(nfp_prog, meta, env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

661

return nfp_bpf_check_store(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

664

return nfp_bpf_check_atomic(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

667

return nfp_bpf_check_alu(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

673

nfp_assign_subprog_idx_and_regs(struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

690

pr_vlog(env, "BUG: number of processed BPF functions is not consistent (processed %d, expected %d)\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

763

int nfp_bpf_finalize(struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

771

nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

772

nfp_prog->subprog_cnt = env->subprog_cnt;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

778

nfp_assign_subprog_idx_and_regs(env, nfp_prog);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

780

info = env->subprog_info;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

794

nn = netdev_priv(env->prog->aux->offload->netdev);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

798

pr_vlog(env, "stack too large: program %dB > FW stack %dB\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

803

nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

807

int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

810

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

811

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

828

pr_vlog(env, "branch hard wire at %d changes target %d -> %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

83

static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

836

pr_vlog(env, "unsupported instruction replacement %hhx -> %hhx\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

841

int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

843

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

844

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

85

const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

86

const struct bpf_reg_state *reg3 = cur_regs(env) + BPF_REG_3;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

92

state = env->cur_state->frame[reg3->frameno];

drivers/net/netdevsim/bpf.c

26

#define pr_vlog(env, fmt, ...) \

drivers/net/netdevsim/bpf.c

27

bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)

drivers/net/netdevsim/bpf.c

63

nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)

drivers/net/netdevsim/bpf.c

68

state = env->prog->aux->offload->dev_priv;

drivers/net/netdevsim/bpf.c

72

if (insn_idx == env->prog->len - 1) {

drivers/net/netdevsim/bpf.c

73

pr_vlog(env, "Hello from netdevsim!\n");

drivers/net/netdevsim/bpf.c

82

static int nsim_bpf_finalize(struct bpf_verifier_env *env)

drivers/net/phy/mdio_bus_provider.c

274

static int mdio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/net/phy/mdio_bus_provider.c

279

rc = of_device_uevent_modalias(dev, env);

drivers/net/wireless/mediatek/mt76/mt7921/mcu.c

1319

u8 env;

drivers/net/wireless/mediatek/mt76/mt7921/mcu.c

1330

.env = env_cap,

drivers/net/wireless/mediatek/mt76/mt7925/mcu.c

3472

u8 env;

drivers/net/wireless/mediatek/mt76/mt7925/mcu.c

3482

.env = env_cap,

drivers/net/wireless/realtek/rtw89/mac80211.c

1957

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/mac80211.c

1960

rpt = list_first_entry_or_null(&env->nhm_rpt_list, typeof(*rpt), list);

drivers/net/wireless/realtek/rtw89/phy.c

6199

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6201

return time_us >> (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6207

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6209

return idx << (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6216

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6219

env->ccx_manual_ctrl = false;

drivers/net/wireless/realtek/rtw89/phy.c

6220

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

6221

env->ccx_rac_lv = RTW89_RAC_RELEASE;

drivers/net/wireless/realtek/rtw89/phy.c

6222

env->ccx_period = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6223

env->ccx_unit_idx = RTW89_CCX_32_US;

drivers/net/wireless/realtek/rtw89/phy.c

6238

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6242

numer = report * score + (env->ccx_period >> 1);

drivers/net/wireless/realtek/rtw89/phy.c

6243

if (env->ccx_period)

drivers/net/wireless/realtek/rtw89/phy.c

6244

ret = numer / env->ccx_period;

drivers/net/wireless/realtek/rtw89/phy.c

6281

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6284

"lv:(%d)->(0)\n", env->ccx_rac_lv);

drivers/net/wireless/realtek/rtw89/phy.c

6286

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

6287

env->ccx_rac_lv = RTW89_RAC_RELEASE;

drivers/net/wireless/realtek/rtw89/phy.c

6288

env->ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;

drivers/net/wireless/realtek/rtw89/phy.c

6295

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6296

bool is_update = env->ifs_clm_app != para->ifs_clm_app;

drivers/net/wireless/realtek/rtw89/phy.c

6298

u16 *ifs_th_l = env->ifs_clm_th_l;

drivers/net/wireless/realtek/rtw89/phy.c

6299

u16 *ifs_th_h = env->ifs_clm_th_h;

drivers/net/wireless/realtek/rtw89/phy.c

6350

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6355

env->ifs_clm_th_l[0], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6357

env->ifs_clm_th_l[1], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6359

env->ifs_clm_th_l[2], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6361

env->ifs_clm_th_l[3], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6364

env->ifs_clm_th_h[0], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6366

env->ifs_clm_th_h[1], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6368

env->ifs_clm_th_h[2], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6370

env->ifs_clm_th_h[3], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6375

i + 1, env->ifs_clm_th_l[i], env->ifs_clm_th_h[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6382

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6385

env->nhm_include_cca = false;

drivers/net/wireless/realtek/rtw89/phy.c

6386

env->nhm_mntr_time = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6387

env->nhm_sum = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6410

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6414

env->ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;

drivers/net/wireless/realtek/rtw89/phy.c

6415

env->ifs_clm_mntr_time = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6437

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6447

"ccx_ongoing=%d, level:(%d)->(%d)\n", env->ccx_ongoing,

drivers/net/wireless/realtek/rtw89/phy.c

6448

env->ccx_rac_lv, level);

drivers/net/wireless/realtek/rtw89/phy.c

6450

if (env->ccx_ongoing) {

drivers/net/wireless/realtek/rtw89/phy.c

6451

if (level <= env->ccx_rac_lv)

drivers/net/wireless/realtek/rtw89/phy.c

6454

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

6458

env->ccx_rac_lv = level;

drivers/net/wireless/realtek/rtw89/phy.c

6470

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6489

env->ccx_ongoing = true;

drivers/net/wireless/realtek/rtw89/phy.c

6495

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6499

env->ifs_clm_tx_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6500

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_tx, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6501

env->ifs_clm_edcca_excl_cca_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6502

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_edcca_excl_cca,

drivers/net/wireless/realtek/rtw89/phy.c

6504

env->ifs_clm_cck_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6505

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6506

env->ifs_clm_ofdm_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6507

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6508

env->ifs_clm_cck_cca_excl_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6509

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckcca_excl_fa,

drivers/net/wireless/realtek/rtw89/phy.c

6511

env->ifs_clm_ofdm_cca_excl_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6512

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmcca_excl_fa,

drivers/net/wireless/realtek/rtw89/phy.c

6514

env->ifs_clm_cck_fa_permil =

drivers/net/wireless/realtek/rtw89/phy.c

6515

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERMIL);

drivers/net/wireless/realtek/rtw89/phy.c

6516

env->ifs_clm_ofdm_fa_permil =

drivers/net/wireless/realtek/rtw89/phy.c

6517

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERMIL);

drivers/net/wireless/realtek/rtw89/phy.c

6520

if (env->ifs_clm_his[i] > ENV_MNTR_IFSCLM_HIS_MAX) {

drivers/net/wireless/realtek/rtw89/phy.c

6521

env->ifs_clm_ifs_avg[i] = ENV_MNTR_FAIL_DWORD;

drivers/net/wireless/realtek/rtw89/phy.c

6523

env->ifs_clm_ifs_avg[i] =

drivers/net/wireless/realtek/rtw89/phy.c

6525

env->ifs_clm_avg[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6528

res = rtw89_phy_ccx_idx_to_us(rtwdev, bb, env->ifs_clm_cca[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6529

res += env->ifs_clm_his[i] >> 1;

drivers/net/wireless/realtek/rtw89/phy.c

6530

if (env->ifs_clm_his[i])

drivers/net/wireless/realtek/rtw89/phy.c

6531

res /= env->ifs_clm_his[i];

drivers/net/wireless/realtek/rtw89/phy.c

6534

env->ifs_clm_cca_avg[i] = res;

drivers/net/wireless/realtek/rtw89/phy.c

6539

env->ifs_clm_tx_ratio, env->ifs_clm_edcca_excl_cca_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6542

env->ifs_clm_cck_fa_ratio, env->ifs_clm_ofdm_fa_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6545

env->ifs_clm_cck_fa_permil, env->ifs_clm_ofdm_fa_permil);

drivers/net/wireless/realtek/rtw89/phy.c

6548

env->ifs_clm_cck_cca_excl_fa_ratio,

drivers/net/wireless/realtek/rtw89/phy.c

6549

env->ifs_clm_ofdm_cca_excl_fa_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6554

i + 1, env->ifs_clm_his[i], env->ifs_clm_ifs_avg[i],

drivers/net/wireless/realtek/rtw89/phy.c

6555

env->ifs_clm_cca_avg[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6560

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6566

if (env->nhm_sum == 0)

drivers/net/wireless/realtek/rtw89/phy.c

6569

weight_zero = clamp_t(u16, env->nhm_th[0] - RTW89_NHM_WEIGHT_OFFSET, 0, U8_MAX);

drivers/net/wireless/realtek/rtw89/phy.c

6575

nhm_weight[i] = env->nhm_th[i - 1] + RTW89_NHM_WEIGHT_OFFSET;

drivers/net/wireless/realtek/rtw89/phy.c

6577

nhm_weight[i] = (env->nhm_th[i - 1] + env->nhm_th[i]) / 2;

drivers/net/wireless/realtek/rtw89/phy.c

6582

if (env->nhm_th[RTW89_NHM_TH_NUM - 1] == RTW89_NHM_WA_TH) {

drivers/net/wireless/realtek/rtw89/phy.c

6584

env->nhm_th[RTW89_NHM_TH_NUM - 2] +

drivers/net/wireless/realtek/rtw89/phy.c

6590

env->nhm_result[0] += env->nhm_result[RTW89_NHM_RPT_NUM - 1];

drivers/net/wireless/realtek/rtw89/phy.c

6591

env->nhm_result[RTW89_NHM_RPT_NUM - 1] = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6595

nhm_weighted_sum += env->nhm_result[i] * nhm_weight[i];

drivers/net/wireless/realtek/rtw89/phy.c

6597

return (nhm_weighted_sum / env->nhm_sum) >> RTW89_NHM_TH_FACTOR;

drivers/net/wireless/realtek/rtw89/phy.c

6605

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6624

env->nhm_result[i] =

drivers/net/wireless/realtek/rtw89/phy.c

6627

sum += env->nhm_result[i];

drivers/net/wireless/realtek/rtw89/phy.c

6629

env->nhm_sum = sum;

drivers/net/wireless/realtek/rtw89/phy.c

6649

rpt = &env->nhm_his[hw_band][chan_idx];

drivers/net/wireless/realtek/rtw89/phy.c

6650

nhm_list = &env->nhm_rpt_list;

drivers/net/wireless/realtek/rtw89/phy.c

6681

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6692

env->ifs_clm_tx =

drivers/net/wireless/realtek/rtw89/phy.c

6695

env->ifs_clm_edcca_excl_cca =

drivers/net/wireless/realtek/rtw89/phy.c

6698

env->ifs_clm_cckcca_excl_fa =

drivers/net/wireless/realtek/rtw89/phy.c

6701

env->ifs_clm_ofdmcca_excl_fa =

drivers/net/wireless/realtek/rtw89/phy.c

6704

env->ifs_clm_cckfa =

drivers/net/wireless/realtek/rtw89/phy.c

6707

env->ifs_clm_ofdmfa =

drivers/net/wireless/realtek/rtw89/phy.c

6711

env->ifs_clm_his[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6714

env->ifs_clm_his[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6718

env->ifs_clm_his[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6721

env->ifs_clm_his[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6725

env->ifs_clm_avg[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6728

env->ifs_clm_avg[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6731

env->ifs_clm_avg[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6734

env->ifs_clm_avg[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6738

env->ifs_clm_cca[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6741

env->ifs_clm_cca[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6744

env->ifs_clm_cca[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6747

env->ifs_clm_cca[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6751

env->ifs_clm_total_ifs =

drivers/net/wireless/realtek/rtw89/phy.c

6756

env->ifs_clm_total_ifs);

drivers/net/wireless/realtek/rtw89/phy.c

6759

env->ifs_clm_tx, env->ifs_clm_edcca_excl_cca);

drivers/net/wireless/realtek/rtw89/phy.c

6762

env->ifs_clm_cckfa, env->ifs_clm_ofdmfa);

drivers/net/wireless/realtek/rtw89/phy.c

6765

env->ifs_clm_cckcca_excl_fa, env->ifs_clm_ofdmcca_excl_fa);

drivers/net/wireless/realtek/rtw89/phy.c

6770

"T%d:[%d, %d, %d]\n", i + 1, env->ifs_clm_his[i],

drivers/net/wireless/realtek/rtw89/phy.c

6771

env->ifs_clm_avg[i], env->ifs_clm_cca[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6782

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6796

env->edcca_clm_ratio = rtw89_phy_ccx_get_report(rtwdev, bb, val, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6804

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6813

env->nhm_th[i] = nhm_th_11k[i] << RTW89_NHM_TH_FACTOR;

drivers/net/wireless/realtek/rtw89/phy.c

6817

env->nhm_th[RTW89_NHM_TH_NUM - 1] = RTW89_NHM_WA_TH;

drivers/net/wireless/realtek/rtw89/phy.c

6823

env->nhm_th[i], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6832

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6850

if (para->mntr_time != env->nhm_mntr_time) {

drivers/net/wireless/realtek/rtw89/phy.c

6858

env->nhm_mntr_time = para->mntr_time;

drivers/net/wireless/realtek/rtw89/phy.c

6859

env->ccx_period = period;

drivers/net/wireless/realtek/rtw89/phy.c

6860

env->ccx_unit_idx = unit_idx;

drivers/net/wireless/realtek/rtw89/phy.c

6863

if (para->nhm_incld_cca != env->nhm_include_cca) {

drivers/net/wireless/realtek/rtw89/phy.c

6868

env->nhm_include_cca = para->nhm_incld_cca;

drivers/net/wireless/realtek/rtw89/phy.c

6907

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6921

if (para->mntr_time != env->ifs_clm_mntr_time) {

drivers/net/wireless/realtek/rtw89/phy.c

6932

env->ifs_clm_mntr_time, para->mntr_time);

drivers/net/wireless/realtek/rtw89/phy.c

6934

env->ifs_clm_mntr_time = para->mntr_time;

drivers/net/wireless/realtek/rtw89/phy.c

6935

env->ccx_period = (u16)period;

drivers/net/wireless/realtek/rtw89/phy.c

6936

env->ccx_unit_idx = (u8)unit_idx;

drivers/net/wireless/realtek/rtw89/phy.c

6940

env->ifs_clm_app = para->ifs_clm_app;

drivers/net/wireless/realtek/rtw89/phy.c

6950

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6954

env->ccx_watchdog_result = RTW89_PHY_ENV_MON_CCX_FAIL;

drivers/net/wireless/realtek/rtw89/phy.c

6955

if (env->ccx_manual_ctrl) {

drivers/net/wireless/realtek/rtw89/phy.c

6966

env->ccx_watchdog_result |= RTW89_PHY_ENV_MON_IFS_CLM;

drivers/net/wireless/realtek/rtw89/phy.c

6969

env->ccx_watchdog_result |= RTW89_PHY_ENV_MON_EDCCA_CLM;

drivers/net/wireless/realtek/rtw89/phy.c

6983

env->ccx_watchdog_result, chk_result);

drivers/net/wireless/realtek/rtw89/phy.c

7391

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

7396

fa_ratio = env->ifs_clm_cck_fa_permil + env->ifs_clm_ofdm_fa_permil;

drivers/net/wireless/realtek/rtw89/phy.c

7424

env->ifs_clm_cck_fa_permil, env->ifs_clm_ofdm_fa_permil,

drivers/net/wireless/realtek/rtw89/phy.c

7425

env->ifs_clm_cck_fa_permil + env->ifs_clm_ofdm_fa_permil,

drivers/net/wireless/realtek/rtw89/phy.c

8199

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

8215

env->nhm_his[hw_band] =

drivers/net/wireless/realtek/rtw89/phy.c

8217

sizeof(*env->nhm_his[0]), GFP_KERNEL);

drivers/net/wireless/realtek/rtw89/phy.c

8220

INIT_LIST_HEAD(&env->nhm_his[hw_band][idx].list);

drivers/net/wireless/realtek/rtw89/phy.c

8222

INIT_LIST_HEAD(&env->nhm_rpt_list);

drivers/net/wireless/realtek/rtw89/phy.c

8757

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

8764

env->edcca_clm_ratio <= RTW89_DIAG_BB_EDCCA_RATIO_TH)

drivers/net/wireless/realtek/rtw89/sar.c

631

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/sar.c

633

u8 tx_ratio = env->ifs_clm_tx_ratio;

drivers/net/xen-netback/xenbus.c

204

struct kobj_uevent_env *env)

drivers/net/xen-netback/xenbus.c

211

if (add_uevent_var(env, "script=%s", be->hotplug_script))

drivers/net/xen-netback/xenbus.c

217

return add_uevent_var(env, "vif=%s", be->vif->dev->name);

drivers/nvdimm/bus.c

47

static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/nvdimm/bus.c

49

return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,

drivers/nvme/host/core.c

134

static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/nvme/host/core.c

4680

static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/nvme/host/core.c

4687

ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);

drivers/nvme/host/core.c

4692

ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);

drivers/nvme/host/core.c

4696

ret = add_uevent_var(env, "NVME_TRSVCID=%s",

drivers/nvme/host/core.c

4701

ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",

drivers/nvme/host/core.c

4706

ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",

drivers/nvmem/layouts.c

49

struct kobj_uevent_env *env)

drivers/nvmem/layouts.c

53

ret = of_device_uevent_modalias(dev, env);

drivers/of/device.c

214

void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/of/device.c

224

add_uevent_var(env, "OF_NAME=%pOFn", dev->of_node);

drivers/of/device.c

225

add_uevent_var(env, "OF_FULLNAME=%pOF", dev->of_node);

drivers/of/device.c

228

add_uevent_var(env, "OF_TYPE=%s", type);

drivers/of/device.c

234

add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);

drivers/of/device.c

237

add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);

drivers/of/device.c

243

add_uevent_var(env, "OF_ALIAS_%d=%s", seen,

drivers/of/device.c

252

int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/of/device.c

260

if (add_uevent_var(env, "MODALIAS="))

drivers/of/device.c

263

sl = of_modalias(dev->of_node, &env->buf[env->buflen-1],

drivers/of/device.c

264

sizeof(env->buf) - env->buflen);

drivers/of/device.c

267

if (sl >= (sizeof(env->buf) - env->buflen))

drivers/of/device.c

269

env->buflen += sl;

drivers/pci/pci-driver.c

1587

static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pci/pci-driver.c

1596

if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))

drivers/pci/pci-driver.c

1599

if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))

drivers/pci/pci-driver.c

1602

if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,

drivers/pci/pci-driver.c

1606

if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))

drivers/pci/pci-driver.c

1609

if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",

drivers/pcmcia/cs.c

798

struct kobj_uevent_env *env)

drivers/pcmcia/cs.c

802

if (add_uevent_var(env, "SOCKET_NO=%u", s->sock))

drivers/pcmcia/ds.c

934

static int pcmcia_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pcmcia/ds.c

952

if (add_uevent_var(env, "SOCKET_NO=%u", p_dev->socket->sock))

drivers/pcmcia/ds.c

955

if (add_uevent_var(env, "DEVICE_NO=%02X", p_dev->device_no))

drivers/pcmcia/ds.c

958

if (add_uevent_var(env, "MODALIAS=pcmcia:m%04Xc%04Xf%02Xfn%02Xpfn%02X"

drivers/platform/chrome/cros_ec_lpc.c

446

static const char *env[] = { "ERROR=PANIC", NULL };

drivers/platform/chrome/cros_ec_lpc.c

456

kobject_uevent_env(&ec_dev->dev->kobj, KOBJ_CHANGE, (char **)env);

drivers/platform/raspberrypi/vchiq-interface/vchiq_bus.c

26

static int vchiq_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/raspberrypi/vchiq-interface/vchiq_bus.c

30

return add_uevent_var(env, "MODALIAS=vchiq:%s", dev_name(&device->dev));

drivers/platform/surface/aggregator/bus.c

41

static int ssam_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/surface/aggregator/bus.c

45

return add_uevent_var(env, "MODALIAS=ssam:d%02Xc%02Xt%02Xi%02Xf%02X",

drivers/platform/wmi/core.c

954

static int wmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/wmi/core.c

958

if (add_uevent_var(env, "MODALIAS=wmi:%pUL", &wblock->gblock.guid))

drivers/platform/wmi/core.c

961

if (add_uevent_var(env, "WMI_GUID=%pUL", &wblock->gblock.guid))

drivers/platform/x86/asus-wmi.c

1885

static int kbd_led_read(struct asus_wmi *asus, int *level, int *env)

drivers/platform/x86/asus-wmi.c

1907

if (env)

drivers/platform/x86/asus-wmi.c

1908

*env = (retval >> 8) & 0x7F;

drivers/pnp/driver.c

153

static int pnp_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pnp/driver.c

163

if (add_uevent_var(env, "MODALIAS=pnp:d%s", pos->id))

drivers/power/supply/power_supply.h

42

extern int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/power/supply/power_supply_sysfs.c

530

static int add_prop_uevent(const struct device *dev, struct kobj_uevent_env *env,

drivers/power/supply/power_supply_sysfs.c

557

return add_uevent_var(env, "POWER_SUPPLY_%s=%s",

drivers/power/supply/power_supply_sysfs.c

561

int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/power/supply/power_supply_sysfs.c

572

ret = add_uevent_var(env, "POWER_SUPPLY_NAME=%s", psy->desc->name);

drivers/power/supply/power_supply_sysfs.c

587

ret = add_prop_uevent(dev, env, POWER_SUPPLY_PROP_TYPE, prop_buf);

drivers/power/supply/power_supply_sysfs.c

592

ret = add_prop_uevent(dev, env, j, prop_buf);

drivers/rapidio/rio-driver.c

207

static int rio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/rapidio/rio-driver.c

218

if (add_uevent_var(env, "MODALIAS=rapidio:v%04Xd%04Xav%04Xad%04X",

drivers/rpmsg/rpmsg_core.c

415

static int rpmsg_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/rpmsg/rpmsg_core.c

420

ret = of_device_uevent_modalias(dev, env);

drivers/rpmsg/rpmsg_core.c

424

return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT,

drivers/s390/cio/css.c

1377

static int css_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/css.c

1382

ret = add_uevent_var(env, "ST=%01X", sch->st);

drivers/s390/cio/css.c

1385

ret = add_uevent_var(env, "MODALIAS=css:t%01X", sch->st);

drivers/s390/cio/device.c

103

static int ccw_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/device.c

111

ret = add_uevent_var(env, "CU_TYPE=%04X", id->cu_type);

drivers/s390/cio/device.c

116

ret = add_uevent_var(env, "CU_MODEL=%02X", id->cu_model);

drivers/s390/cio/device.c

122

ret = add_uevent_var(env, "DEV_TYPE=%04X", id->dev_type);

drivers/s390/cio/device.c

127

ret = add_uevent_var(env, "DEV_MODEL=%02X", id->dev_model);

drivers/s390/cio/device.c

133

ret = add_uevent_var(env, "MODALIAS=%s", modalias_buf);

drivers/s390/cio/scm.c

41

static int scmdev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/scm.c

43

return add_uevent_var(env, "MODALIAS=scm:scmdev");

drivers/s390/crypto/ap_bus.c

661

static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/crypto/ap_bus.c

674

rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);

drivers/s390/crypto/ap_bus.c

678

rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);

drivers/s390/crypto/ap_bus.c

684

rc = add_uevent_var(env, "MODE=accel");

drivers/s390/crypto/ap_bus.c

686

rc = add_uevent_var(env, "MODE=cca");

drivers/s390/crypto/ap_bus.c

688

rc = add_uevent_var(env, "MODE=ep11");

drivers/s390/crypto/ap_bus.c

696

rc = add_uevent_var(env, "MODE=accel");

drivers/s390/crypto/ap_bus.c

698

rc = add_uevent_var(env, "MODE=cca");

drivers/s390/crypto/ap_bus.c

700

rc = add_uevent_var(env, "MODE=ep11");

drivers/s390/net/qeth_l2_main.c

1254

char *env[8];

drivers/s390/net/qeth_l2_main.c

1262

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1266

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1271

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1275

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1277

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1280

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1283

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1287

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1291

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1294

env[i] = NULL;

drivers/s390/net/qeth_l2_main.c

1295

kobject_uevent_env(&card->gdev->dev.kobj, KOBJ_CHANGE, env);

drivers/s390/net/qeth_l2_main.c

1312

char *env[] = {

drivers/s390/net/qeth_l2_main.c

1331

KOBJ_CHANGE, env);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1289

struct kobj_uevent_env *env;

drivers/scsi/mpi3mr/mpi3mr_fw.c

1292

env = kzalloc_obj(*env);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1293

if (!env)

drivers/scsi/mpi3mr/mpi3mr_fw.c

1296

ret = add_uevent_var(env, "DRIVER=%s", mrioc->driver_name);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1300

ret = add_uevent_var(env, "IOC_ID=%u", mrioc->id);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1304

ret = add_uevent_var(env, "FAULT_CODE=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1309

ret = add_uevent_var(env, "FAULT_INFO0=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1314

ret = add_uevent_var(env, "FAULT_INFO1=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1319

ret = add_uevent_var(env, "FAULT_INFO2=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1325

KOBJ_CHANGE, env->envp);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1328

kfree(env);

drivers/scsi/pm8001/pm80xx_hwi.c

1569

struct kobj_uevent_env *env;

drivers/scsi/pm8001/pm80xx_hwi.c

1573

env = kzalloc_obj(struct kobj_uevent_env);

drivers/scsi/pm8001/pm80xx_hwi.c

1574

if (!env)

drivers/scsi/pm8001/pm80xx_hwi.c

1577

if (add_uevent_var(env, "DRIVER=%s", DRV_NAME))

drivers/scsi/pm8001/pm80xx_hwi.c

1580

if (add_uevent_var(env, "HBA_NUM=%u", pm8001_ha->id))

drivers/scsi/pm8001/pm80xx_hwi.c

1583

if (add_uevent_var(env, "EVENT_TYPE=FATAL_ERROR"))

drivers/scsi/pm8001/pm80xx_hwi.c

1588

if (add_uevent_var(env, "REPORTED_BY=DRIVER"))

drivers/scsi/pm8001/pm80xx_hwi.c

1592

if (add_uevent_var(env, "REPORTED_BY=FIRMWARE"))

drivers/scsi/pm8001/pm80xx_hwi.c

1596

if (add_uevent_var(env, "REPORTED_BY=OTHER"))

drivers/scsi/pm8001/pm80xx_hwi.c

1601

kobject_uevent_env(&pm8001_ha->shost->shost_dev.kobj, KOBJ_CHANGE, env->envp);

drivers/scsi/pm8001/pm80xx_hwi.c

1604

kfree(env);

drivers/scsi/scsi_sysfs.c

544

static int scsi_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/scsi/scsi_sysfs.c

553

add_uevent_var(env, "MODALIAS=" SCSI_DEVICE_MODALIAS_FMT, sdev->type);

drivers/slimbus/core.c

100

return add_uevent_var(env, "MODALIAS=slim:%s", dev_name(&sbdev->dev));

drivers/slimbus/core.c

96

static int slim_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soc/qcom/apr.c

390

static int apr_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soc/qcom/apr.c

395

ret = of_device_uevent_modalias(dev, env);

drivers/soc/qcom/apr.c

399

return add_uevent_var(env, "MODALIAS=apr:%s", adev->name);

drivers/soundwire/bus_type.c

61

int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soundwire/bus_type.c

68

if (add_uevent_var(env, "MODALIAS=%s", modalias))

drivers/spi/spi.c

395

static int spi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/spi/spi.c

400

rc = acpi_device_uevent_modalias(dev, env);

drivers/spi/spi.c

404

return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);

drivers/spmi/spmi.c

370

static int spmi_drv_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/spmi/spmi.c

374

ret = of_device_uevent_modalias(dev, env);

drivers/ssb/main.c

342

static int ssb_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ssb/main.c

351

return add_uevent_var(env,

drivers/staging/greybus/gbphy.c

74

static int gbphy_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/staging/greybus/gbphy.c

83

if (add_uevent_var(env, "BUS=%u", hd->bus_id))

drivers/staging/greybus/gbphy.c

85

if (add_uevent_var(env, "MODULE=%u", module->module_id))

drivers/staging/greybus/gbphy.c

87

if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))

drivers/staging/greybus/gbphy.c

89

if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",

drivers/staging/greybus/gbphy.c

92

if (add_uevent_var(env, "BUNDLE=%u", gbphy_dev->bundle->id))

drivers/staging/greybus/gbphy.c

94

if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))

drivers/staging/greybus/gbphy.c

96

if (add_uevent_var(env, "GBPHY=%u", gbphy_dev->id))

drivers/staging/greybus/gbphy.c

98

if (add_uevent_var(env, "PROTOCOL_ID=%02x", cport_desc->protocol_id))

drivers/staging/media/atomisp/pci/ia_css_control.h

37

const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/ia_css_device_access.c

16

ia_css_device_access_init(const struct ia_css_hw_access_env *env)

drivers/staging/media/atomisp/pci/ia_css_device_access.c

18

assert(env);

drivers/staging/media/atomisp/pci/ia_css_device_access.c

20

my_env = *env;

drivers/staging/media/atomisp/pci/ia_css_device_access.h

19

ia_css_device_access_init(const struct ia_css_hw_access_env *env);

drivers/staging/media/atomisp/pci/ia_css_firmware.h

42

ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1308

ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1313

if (!env)

drivers/staging/media/atomisp/pci/sh_css.c

1321

if (my_css.flush != env->cpu_mem_env.flush) {

drivers/staging/media/atomisp/pci/sh_css.c

1323

my_css.flush = env->cpu_mem_env.flush;

drivers/staging/media/atomisp/pci/sh_css.c

1335

ia_css_init(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1343

if (!env)

drivers/staging/media/atomisp/pci/sh_css.c

1346

sh_css_printf = env->print_env.debug_print;

drivers/staging/media/atomisp/pci/sh_css.c

1350

flush_func = env->cpu_mem_env.flush;

drivers/staging/media/atomisp/pci/sh_css.c

1356

ia_css_device_access_init(&env->hw_access_env);

drivers/staging/media/atomisp/pci/sh_css.c

1366

my_css_save.driver_env = *env;

drivers/staging/media/ipu3/ipu3-css.c

1699

struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];

drivers/staging/media/ipu3/ipu3-css.c

1784

env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;

drivers/staging/media/ipu3/ipu3-css.c

1786

env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;

drivers/tee/tee_core.c

1434

struct kobj_uevent_env *env)

drivers/tee/tee_core.c

1438

return add_uevent_var(env, "MODALIAS=tee:%pUb", dev_id);

drivers/thunderbolt/switch.c

2309

static int tb_switch_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/thunderbolt/switch.c

2315

if (add_uevent_var(env, "USB4_VERSION=%u.0",

drivers/thunderbolt/switch.c

2338

if (add_uevent_var(env, "USB4_TYPE=%s", type))

drivers/thunderbolt/xdomain.c

1111

static int tb_service_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/thunderbolt/xdomain.c

1117

return add_uevent_var(env, "MODALIAS=%s", modalias);

drivers/tty/serdev/core.c

48

static int serdev_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/tty/serdev/core.c

54

rc = acpi_device_uevent_modalias(dev, env);

drivers/tty/serdev/core.c

58

return of_device_uevent_modalias(dev, env);

drivers/usb/common/ulpi.c

58

static int ulpi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/common/ulpi.c

63

ret = of_device_uevent_modalias(dev, env);

drivers/usb/common/ulpi.c

67

if (add_uevent_var(env, "MODALIAS=ulpi:v%04xp%04x",

drivers/usb/core/driver.c

915

static int usb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/driver.c

940

if (add_uevent_var(env, "PRODUCT=%x/%x/%x",

drivers/usb/core/driver.c

947

if (add_uevent_var(env, "TYPE=%d/%d/%d",

drivers/usb/core/hcd.c

2489

static char *env[] = {

drivers/usb/core/hcd.c

2495

kobject_uevent_env(&hcd->self.root_hub->dev.kobj, KOBJ_OFFLINE, env);

drivers/usb/core/message.c

1881

static int usb_if_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/message.c

1891

if (add_uevent_var(env, "INTERFACE=%d/%d/%d",

drivers/usb/core/message.c

1897

if (add_uevent_var(env,

drivers/usb/core/usb.c

504

static int usb_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/usb.c

510

if (add_uevent_var(env, "BUSNUM=%03d", usb_dev->bus->busnum))

drivers/usb/core/usb.c

513

if (add_uevent_var(env, "DEVNUM=%03d", usb_dev->devnum))

drivers/usb/gadget/udc/core.c

1922

static int usb_udc_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/gadget/udc/core.c

1927

ret = add_uevent_var(env, "USB_UDC_NAME=%s", udc->gadget->name);

drivers/usb/gadget/udc/core.c

1935

ret = add_uevent_var(env, "USB_UDC_DRIVER=%s",

drivers/usb/phy/phy.c

148

static int usb_phy_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/phy/phy.c

168

if (add_uevent_var(env, uchger_state))

drivers/usb/phy/phy.c

171

if (add_uevent_var(env, uchger_type))

drivers/usb/roles/class.c

321

static int usb_role_switch_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/roles/class.c

325

ret = add_uevent_var(env, "USB_ROLE_SWITCH=%s", dev_name(dev));

drivers/usb/typec/bus.c

481

static int typec_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/typec/bus.c

488

if (add_uevent_var(env, "SVID=%04X", altmode->svid))

drivers/usb/typec/bus.c

491

if (add_uevent_var(env, "MODE=%u", altmode->mode))

drivers/usb/typec/bus.c

494

return add_uevent_var(env, "MODALIAS=typec:id%04X", altmode->svid);

drivers/usb/typec/class.c

2099

static int typec_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/typec/class.c

2103

ret = add_uevent_var(env, "TYPEC_PORT=%s", dev_name(dev));

drivers/virtio/virtio.c

102

return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",

drivers/virtio/virtio.c

98

static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env)

drivers/w1/w1.c

168

static int w1_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/w1/w1.c

570

static int w1_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/w1/w1.c

596

err = add_uevent_var(env, "W1_FID=%02X", sl->reg_num.family);

drivers/w1/w1.c

600

err = add_uevent_var(env, "W1_SLAVE_ID=%024LX",

drivers/xen/pvcalls-back.c

1198

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_backend.c

110

if (add_uevent_var(env, "MODALIAS=xen-backend:%s", xdev->devicetype))

drivers/xen/xenbus/xenbus_probe_backend.c

114

if (add_uevent_var(env, "XENBUS_TYPE=%s", xdev->devicetype))

drivers/xen/xenbus/xenbus_probe_backend.c

117

if (add_uevent_var(env, "XENBUS_PATH=%s", xdev->nodename))

drivers/xen/xenbus/xenbus_probe_backend.c

120

if (add_uevent_var(env, "XENBUS_BASE_PATH=%s", bus->root))

drivers/xen/xenbus/xenbus_probe_backend.c

126

return drv->uevent(xdev, env);

drivers/xen/xenbus/xenbus_probe_backend.c

96

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_frontend.c

77

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_frontend.c

81

if (add_uevent_var(env, "MODALIAS=xen:%s", dev->devicetype))

drivers/zorro/zorro-driver.c

133

static int zorro_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/zorro/zorro-driver.c

144

if (add_uevent_var(env, "ZORRO_ID=%08X", z->id) ||

drivers/zorro/zorro-driver.c

145

add_uevent_var(env, "ZORRO_SLOT_NAME=%s", dev_name(dev)) ||

drivers/zorro/zorro-driver.c

146

add_uevent_var(env, "ZORRO_SLOT_ADDR=%04X", z->slotaddr) ||

drivers/zorro/zorro-driver.c

147

add_uevent_var(env, "MODALIAS=" ZORRO_DEVICE_MODALIAS_FMT, z->id))

fs/dlm/lockspace.c

214

static int dlm_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/dlm/lockspace.c

218

add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name);

fs/fuse/virtio_fs.c

1790

static int virtio_fs_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/fuse/virtio_fs.c

1794

add_uevent_var(env, "TAG=%s", fs->tag);

fs/gfs2/sys.c

767

static int gfs2_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/gfs2/sys.c

772

add_uevent_var(env, "LOCKTABLE=%s", sdp->sd_table_name);

fs/gfs2/sys.c

773

add_uevent_var(env, "LOCKPROTO=%s", sdp->sd_proto_name);

fs/gfs2/sys.c

775

add_uevent_var(env, "JOURNALID=%d", sdp->sd_lockstruct.ls_jid);

fs/gfs2/sys.c

777

add_uevent_var(env, "UUID=%pUB", &s->s_uuid);

fs/hfsplus/unicode_test.c

23

struct test_mock_string_env *env;

fs/hfsplus/unicode_test.c

25

env = kzalloc_obj(struct test_mock_string_env);

fs/hfsplus/unicode_test.c

26

if (!env)

fs/hfsplus/unicode_test.c

29

env->buf = kzalloc(buf_size, GFP_KERNEL);

fs/hfsplus/unicode_test.c

30

if (!env->buf) {

fs/hfsplus/unicode_test.c

31

kfree(env);

fs/hfsplus/unicode_test.c

35

env->buf_size = buf_size;

fs/hfsplus/unicode_test.c

37

return env;

fs/hfsplus/unicode_test.c

40

static void free_mock_str_env(struct test_mock_string_env *env)

fs/hfsplus/unicode_test.c

42

if (env->buf)

fs/hfsplus/unicode_test.c

43

kfree(env->buf);

fs/hfsplus/unicode_test.c

44

kfree(env);

include/linux/acpi.h

1030

struct kobj_uevent_env *env)

include/linux/bpf.h

1171

int (*insn_hook)(struct bpf_verifier_env *env,

include/linux/bpf.h

1173

int (*finalize)(struct bpf_verifier_env *env);

include/linux/bpf.h

1175

int (*replace_insn)(struct bpf_verifier_env *env, u32 off,

include/linux/bpf.h

1177

int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);

include/linux/bpf.h

1603

bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,

include/linux/bpf.h

1605

u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog);

include/linux/bpf.h

177

int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,

include/linux/bpf.h

3021

int map_set_for_each_callback_args(struct bpf_verifier_env *env,

include/linux/bpf.h

3158

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);

include/linux/bpf_verifier.h

1021

static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

1023

return &env->prog->aux->func_info_aux[subprog];

include/linux/bpf_verifier.h

1026

static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

1028

return &env->subprog_info[subprog];

include/linux/bpf_verifier.h

1057

__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1066

__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1070

#define verifier_bug_if(cond, env, fmt, args...) \

include/linux/bpf_verifier.h

1074

verifier_bug(env, fmt " (" #cond ")", ##args); \

include/linux/bpf_verifier.h

1077

#define verifier_bug(env, fmt, args...) \

include/linux/bpf_verifier.h

1080

bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \

include/linux/bpf_verifier.h

1083

static inline void mark_prune_point(struct bpf_verifier_env *env, int idx)

include/linux/bpf_verifier.h

1085

env->insn_aux_data[idx].prune_point = true;

include/linux/bpf_verifier.h

1088

static inline bool bpf_is_prune_point(struct bpf_verifier_env *env, int insn_idx)

include/linux/bpf_verifier.h

1090

return env->insn_aux_data[insn_idx].prune_point;

include/linux/bpf_verifier.h

1093

static inline void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)

include/linux/bpf_verifier.h

1095

env->insn_aux_data[idx].force_checkpoint = true;

include/linux/bpf_verifier.h

1098

static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)

include/linux/bpf_verifier.h

1100

return env->insn_aux_data[insn_idx].force_checkpoint;

include/linux/bpf_verifier.h

1103

static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx)

include/linux/bpf_verifier.h

1105

env->insn_aux_data[idx].calls_callback = true;

include/linux/bpf_verifier.h

1108

static inline bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)

include/linux/bpf_verifier.h

1110

return env->insn_aux_data[insn_idx].calls_callback;

include/linux/bpf_verifier.h

1113

static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)

include/linux/bpf_verifier.h

1115

env->insn_aux_data[idx].jmp_point = true;

include/linux/bpf_verifier.h

1118

static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1120

struct bpf_verifier_state *cur = env->cur_state;

include/linux/bpf_verifier.h

1125

static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1127

return cur_func(env)->regs;

include/linux/bpf_verifier.h

1131

int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1133

int bpf_prog_offload_finalize(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1135

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

include/linux/bpf_verifier.h

1138

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);

include/linux/bpf_verifier.h

1159

int bpf_check_btf_info_early(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1161

int bpf_check_btf_info(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1171

int mark_chain_precision(struct bpf_verifier_env *env, int regno);

include/linux/bpf_verifier.h

1173

int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx);

include/linux/bpf_verifier.h

1174

int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

include/linux/bpf_verifier.h

1179

struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);

include/linux/bpf_verifier.h

1182

int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,

include/linux/bpf_verifier.h

1185

void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1188

void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1190

void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

include/linux/bpf_verifier.h

1191

int bpf_mark_chain_precision(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1200

static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1203

struct bpf_verifier_state *cur = env->cur_state;

include/linux/bpf_verifier.h

1216

static inline bool bpf_is_jmp_point(struct bpf_verifier_env *env, int insn_idx)

include/linux/bpf_verifier.h

1218

return env->insn_aux_data[insn_idx].jmp_point;

include/linux/bpf_verifier.h

1338

static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)

include/linux/bpf_verifier.h

1340

env->scratched_regs |= 1U << regno;

include/linux/bpf_verifier.h

1343

static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)

include/linux/bpf_verifier.h

1345

env->scratched_stack_slots |= 1ULL << spi;

include/linux/bpf_verifier.h

1348

static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)

include/linux/bpf_verifier.h

1350

return (env->scratched_regs >> regno) & 1;

include/linux/bpf_verifier.h

1353

static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)

include/linux/bpf_verifier.h

1355

return (env->scratched_stack_slots >> regno) & 1;

include/linux/bpf_verifier.h

1358

static inline bool verifier_state_scratched(const struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1360

return env->scratched_regs || env->scratched_stack_slots;

include/linux/bpf_verifier.h

1363

static inline void mark_verifier_state_clean(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1365

env->scratched_regs = 0U;

include/linux/bpf_verifier.h

1366

env->scratched_stack_slots = 0ULL;

include/linux/bpf_verifier.h

1370

static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1372

env->scratched_regs = ~0U;

include/linux/bpf_verifier.h

1373

env->scratched_stack_slots = ~0ULL;

include/linux/bpf_verifier.h

1392

const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type);

include/linux/bpf_verifier.h

1397

void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

include/linux/bpf_verifier.h

1399

void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

include/linux/bpf_verifier.h

1403

struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);

include/linux/bpf_verifier.h

1405

struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);

include/linux/bpf_verifier.h

1407

bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);

include/linux/bpf_verifier.h

1409

int bpf_find_subprog(struct bpf_verifier_env *env, int off);

include/linux/bpf_verifier.h

1411

int bpf_compute_const_regs(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1412

int bpf_prune_dead_branches(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1413

int bpf_check_cfg(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1414

int bpf_compute_postorder(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1415

int bpf_compute_scc(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1490

int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,

include/linux/bpf_verifier.h

1492

int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id,

include/linux/bpf_verifier.h

1511

void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);

include/linux/bpf_verifier.h

1512

bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,

include/linux/bpf_verifier.h

1514

s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1517

s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

1520

int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1522

int bpf_stack_liveness_init(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1523

void bpf_stack_liveness_free(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1524

int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

include/linux/bpf_verifier.h

1525

bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);

include/linux/bpf_verifier.h

1526

int bpf_compute_live_registers(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1587

void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);

include/linux/bpf_verifier.h

1588

void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog);

include/linux/bpf_verifier.h

1589

bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1590

bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm);

include/linux/bpf_verifier.h

1591

int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset);

include/linux/bpf_verifier.h

1592

int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

include/linux/bpf_verifier.h

1600

int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1601

int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1602

void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1603

int bpf_opt_remove_dead_code(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1604

int bpf_opt_remove_nops(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1605

int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr);

include/linux/bpf_verifier.h

1606

int bpf_convert_ctx_accesses(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1607

int bpf_jit_subprogs(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1608

int bpf_fixup_call_args(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1609

int bpf_do_misc_fixups(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

841

struct bpf_verifier_env *env;

include/linux/device.h

90

int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/device/bus.h

91

int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/device/class.h

56

int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/filter.h

1134

struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,

include/linux/filter.h

1178

struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);

include/linux/filter.h

1212

struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,

include/linux/filter.h

1214

struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);

include/linux/filter.h

1215

void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,

include/linux/filter.h

1218

static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,

include/linux/filter.h

1224

static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)

include/linux/filter.h

1229

static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,

include/linux/filter.h

1371

struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog);

include/linux/filter.h

1523

struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)

include/linux/ieee80211-he.h

643

const struct ieee80211_tx_pwr_env *env = (const void *)data;

include/linux/ieee80211-he.h

645

u8 needed = sizeof(*env);

include/linux/ieee80211-he.h

651

count = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_COUNT);

include/linux/ieee80211-he.h

652

interpret = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_INTERPRET);

include/linux/ieee80211-he.h

653

category = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_CATEGORY);

include/linux/ieee80211-he.h

690

u8 K = u8_get_bits(env->variable[N],

include/linux/kobject.h

136

int (* const uevent)(const struct kobject *kobj, struct kobj_uevent_env *env);

include/linux/kobject.h

220

int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...);

include/linux/of_device.h

28

extern void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env);

include/linux/of_device.h

29

extern int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env);

include/linux/of_device.h

52

struct kobj_uevent_env *env) { }

include/linux/of_device.h

61

struct kobj_uevent_env *env)

include/linux/soundwire/sdw_type.h

24

int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env);

include/rv/ha_monitor.h

101

static void ha_react(enum states curr_state, enum events event, char *env) { }

include/rv/ha_monitor.h

107

static char *model_get_env_name(enum envs env)

include/rv/ha_monitor.h

109

if ((env < 0) || (env >= ENV_MAX))

include/rv/ha_monitor.h

112

return RV_AUTOMATON_NAME.env_names[env];

include/rv/ha_monitor.h

161

static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs env, u64 time_ns);

include/rv/ha_monitor.h

211

static inline bool ha_monitor_env_invalid(struct ha_monitor *ha_mon, enum envs env)

include/rv/ha_monitor.h

213

return READ_ONCE(ha_mon->env_store[env]) == ENV_INVALID_VALUE;

include/rv/ha_monitor.h

230

char *curr_state, char *event, char *env,

include/rv/ha_monitor.h

233

CONCATENATE(trace_error_env_, MONITOR_NAME)(curr_state, event, env);

include/rv/ha_monitor.h

240

char *curr_state, char *event, char *env,

include/rv/ha_monitor.h

243

CONCATENATE(trace_error_env_, MONITOR_NAME)(id, curr_state, event, env);

include/rv/ha_monitor.h

348

static inline u64 ha_get_clk_ns(struct ha_monitor *ha_mon, enum envs env, u64 time_ns)

include/rv/ha_monitor.h

350

return time_ns - READ_ONCE(ha_mon->env_store[env]);

include/rv/ha_monitor.h

352

static inline void ha_reset_clk_ns(struct ha_monitor *ha_mon, enum envs env, u64 time_ns)

include/rv/ha_monitor.h

354

WRITE_ONCE(ha_mon->env_store[env], time_ns);

include/rv/ha_monitor.h

356

static inline void ha_set_invariant_ns(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

359

WRITE_ONCE(ha_mon->env_store[env], time_ns + value);

include/rv/ha_monitor.h

362

enum envs env, u64 time_ns)

include/rv/ha_monitor.h

364

return READ_ONCE(ha_mon->env_store[env]) >= time_ns;

include/rv/ha_monitor.h

369

static inline u64 ha_invariant_passed_ns(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

374

if (env < 0 || env >= ENV_MAX_STORED)

include/rv/ha_monitor.h

376

if (ha_monitor_env_invalid(ha_mon, env))

include/rv/ha_monitor.h

378

passed = ha_get_env(ha_mon, env, time_ns);

include/rv/ha_monitor.h

379

ha_set_invariant_ns(ha_mon, env, expire - passed, time_ns);

include/rv/ha_monitor.h

386

static inline u64 ha_get_clk_jiffy(struct ha_monitor *ha_mon, enum envs env)

include/rv/ha_monitor.h

388

return get_jiffies_64() - READ_ONCE(ha_mon->env_store[env]);

include/rv/ha_monitor.h

390

static inline void ha_reset_clk_jiffy(struct ha_monitor *ha_mon, enum envs env)

include/rv/ha_monitor.h

392

WRITE_ONCE(ha_mon->env_store[env], get_jiffies_64());

include/rv/ha_monitor.h

395

enum envs env, u64 value)

include/rv/ha_monitor.h

397

WRITE_ONCE(ha_mon->env_store[env], get_jiffies_64() + value);

include/rv/ha_monitor.h

400

enum envs env, u64 time_ns)

include/rv/ha_monitor.h

402

return time_after64(READ_ONCE(ha_mon->env_store[env]), get_jiffies_64());

include/rv/ha_monitor.h

408

static inline u64 ha_invariant_passed_jiffy(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

413

if (env < 0 || env >= ENV_MAX_STORED)

include/rv/ha_monitor.h

415

if (ha_monitor_env_invalid(ha_mon, env))

include/rv/ha_monitor.h

417

passed = ha_get_env(ha_mon, env, time_ns);

include/rv/ha_monitor.h

418

ha_set_invariant_jiffy(ha_mon, env, expire - passed);

include/rv/ha_monitor.h

433

static inline void ha_inv_to_guard(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

436

WRITE_ONCE(ha_mon->env_store[env], READ_ONCE(ha_mon->env_store[env]) - value);

include/rv/ha_monitor.h

463

static inline void ha_start_timer_jiffy(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

466

u64 passed = ha_invariant_passed_jiffy(ha_mon, env, expire, time_ns);

include/rv/ha_monitor.h

470

static inline void ha_start_timer_ns(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

473

u64 passed = ha_invariant_passed_ns(ha_mon, env, expire, time_ns);

include/rv/ha_monitor.h

515

static inline void ha_start_timer_ns(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

519

u64 passed = ha_invariant_passed_ns(ha_mon, env, expire, time_ns);

include/rv/ha_monitor.h

525

static inline void ha_start_timer_jiffy(struct ha_monitor *ha_mon, enum envs env,

include/rv/ha_monitor.h

528

u64 passed = ha_invariant_passed_jiffy(ha_mon, env, expire, time_ns);

include/rv/ha_monitor.h

90

static void ha_react(enum states curr_state, enum events event, char *env)

include/rv/ha_monitor.h

96

model_get_state_name(curr_state), env);

kernel/bpf/backtrack.c

11

int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,

kernel/bpf/backtrack.c

120

struct bpf_verifier_env *env = bt->env;

kernel/bpf/backtrack.c

123

bt->env = env;

kernel/bpf/backtrack.c

150

verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);

kernel/bpf/backtrack.c

160

verifier_bug(bt->env, "subprog exit from frame 0");

kernel/bpf/backtrack.c

19

if (env->cur_hist_ent) {

kernel/bpf/backtrack.c

23

verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&

kernel/bpf/backtrack.c

24

(env->cur_hist_ent->flags & insn_flags) != insn_flags,

kernel/bpf/backtrack.c

25

env, "insn history: insn_idx %d cur flags %x new flags %x",

kernel/bpf/backtrack.c

26

env->insn_idx, env->cur_hist_ent->flags, insn_flags);

kernel/bpf/backtrack.c

267

static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,

kernel/bpf/backtrack.c

27

env->cur_hist_ent->flags |= insn_flags;

kernel/bpf/backtrack.c

270

struct bpf_insn *insn = env->prog->insnsi + idx;

kernel/bpf/backtrack.c

28

env->cur_hist_ent->spi = spi;

kernel/bpf/backtrack.c

280

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

281

fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));

kernel/bpf/backtrack.c

282

verbose(env, "mark_precise: frame%d: regs=%s ",

kernel/bpf/backtrack.c

283

bt->frame, env->tmp_str_buf);

kernel/bpf/backtrack.c

284

bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));

kernel/bpf/backtrack.c

285

verbose(env, "stack=%s before ", env->tmp_str_buf);

kernel/bpf/backtrack.c

286

verbose(env, "%d: ", idx);

kernel/bpf/backtrack.c

287

bpf_verbose_insn(env, insn);

kernel/bpf/backtrack.c

29

env->cur_hist_ent->frame = frame;

kernel/bpf/backtrack.c

30

verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,

kernel/bpf/backtrack.c

32

env->insn_idx, env->cur_hist_ent->linked_regs);

kernel/bpf/backtrack.c

33

env->cur_hist_ent->linked_regs = linked_regs;

kernel/bpf/backtrack.c

414

subprog = bpf_find_subprog(env, subprog_insn_idx);

kernel/bpf/backtrack.c

418

if (bpf_subprog_is_global(env, subprog)) {

kernel/bpf/backtrack.c

425

verifier_bug_if(idx + 1 != subseq_idx, env,

kernel/bpf/backtrack.c

432

verifier_bug(env, "global subprog unexpected regs %x",

kernel/bpf/backtrack.c

447

verifier_bug(env, "static subprog unexpected regs %x",

kernel/bpf/backtrack.c

45

p->idx = env->insn_idx;

kernel/bpf/backtrack.c

455

verifier_bug(env,

kernel/bpf/backtrack.c

46

p->prev_idx = env->prev_insn_idx;

kernel/bpf/backtrack.c

468

verifier_bug(env,

kernel/bpf/backtrack.c

486

verifier_bug(env, "callback unexpected regs %x",

kernel/bpf/backtrack.c

491

verifier_bug(env, "callback leftover stack slots %llx",

kernel/bpf/backtrack.c

514

verifier_bug(env, "backtracking call unexpected regs %x",

kernel/bpf/backtrack.c

52

env->cur_hist_ent = p;

kernel/bpf/backtrack.c

534

if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))

kernel/bpf/backtrack.c

538

verifier_bug(env, "backtracking exit unexpected regs %x",

kernel/bpf/backtrack.c

552

bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&

kernel/bpf/backtrack.c

656

void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,

kernel/bpf/backtrack.c

663

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

664

verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",

kernel/bpf/backtrack.c

682

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

683

verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",

kernel/bpf/backtrack.c

694

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

695

verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",

kernel/bpf/backtrack.c

790

int bpf_mark_chain_precision(struct bpf_verifier_env *env,

kernel/bpf/backtrack.c

796

struct backtrack_state *bt = &env->bt;

kernel/bpf/backtrack.c

8

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/backtrack.c

805

if (!env->bpf_capable)

kernel/bpf/backtrack.c

820

verifier_bug(env, "backtracking misuse");

kernel/bpf/backtrack.c

834

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

835

verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",

kernel/bpf/backtrack.c

862

verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",

kernel/bpf/backtrack.c

873

err = backtrack_insn(env, i, subseq_idx, hist, bt);

kernel/bpf/backtrack.c

876

bpf_mark_all_scalars_precise(env, starting_state);

kernel/bpf/backtrack.c

892

if (i >= env->prog->len) {

kernel/bpf/backtrack.c

899

verifier_bug(env, "backtracking idx %d", i);

kernel/bpf/backtrack.c

927

env, "stack slot %d, total slots %d",

kernel/bpf/backtrack.c

954

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/backtrack.c

955

fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,

kernel/bpf/backtrack.c

957

verbose(env, "mark_precise: frame%d: parent state regs=%s ",

kernel/bpf/backtrack.c

958

fr, env->tmp_str_buf);

kernel/bpf/backtrack.c

959

bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,

kernel/bpf/backtrack.c

961

verbose(env, "stack=%s: ", env->tmp_str_buf);

kernel/bpf/backtrack.c

962

print_verifier_state(env, st, fr, true);

kernel/bpf/backtrack.c

979

bpf_mark_all_scalars_precise(env, starting_state);

kernel/bpf/btf.c

1546

__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

1549

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1560

__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,

kernel/bpf/btf.c

1565

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1566

struct btf *btf = env->btf;

kernel/bpf/btf.c

1581

if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))

kernel/bpf/btf.c

1586

env->log_type_id,

kernel/bpf/btf.c

1592

btf_type_ops(t)->log_details(env, t);

kernel/bpf/btf.c

1604

#define btf_verifier_log_type(env, t, ...) \

kernel/bpf/btf.c

1605

__btf_verifier_log_type((env), (t), true, __VA_ARGS__)

kernel/bpf/btf.c

1606

#define btf_verifier_log_basic(env, t, ...) \

kernel/bpf/btf.c

1607

__btf_verifier_log_type((env), (t), false, __VA_ARGS__)

kernel/bpf/btf.c

1610

static void btf_verifier_log_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

1615

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1616

struct btf *btf = env->btf;

kernel/bpf/btf.c

1627

if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))

kernel/bpf/btf.c

1637

if (env->phase != CHECK_META)

kernel/bpf/btf.c

1638

btf_verifier_log_type(env, struct_type, NULL);

kernel/bpf/btf.c

1663

static void btf_verifier_log_vsi(struct btf_verifier_env *env,

kernel/bpf/btf.c

1668

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1675

if (env->phase != CHECK_META)

kernel/bpf/btf.c

1676

btf_verifier_log_type(env, datasec_type, NULL);

kernel/bpf/btf.c

1690

static void btf_verifier_log_hdr(struct btf_verifier_env *env,

kernel/bpf/btf.c

1693

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1694

const struct btf *btf = env->btf;

kernel/bpf/btf.c

1719

static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)

kernel/bpf/btf.c

1721

struct btf *btf = env->btf;

kernel/bpf/btf.c

1730

btf_verifier_log(env, "Exceeded max num of types");

kernel/bpf/btf.c

1926

static int env_resolve_init(struct btf_verifier_env *env)

kernel/bpf/btf.c

1928

struct btf *btf = env->btf;

kernel/bpf/btf.c

1951

env->visit_states = visit_states;

kernel/bpf/btf.c

1962

static void btf_verifier_env_free(struct btf_verifier_env *env)

kernel/bpf/btf.c

1964

kvfree(env->visit_states);

kernel/bpf/btf.c

1965

kfree(env);

kernel/bpf/btf.c

1968

static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,

kernel/bpf/btf.c

1971

switch (env->resolve_mode) {

kernel/bpf/btf.c

1993

static bool env_type_is_resolved(const struct btf_verifier_env *env,

kernel/bpf/btf.c

1997

if (type_id < env->btf->start_id)

kernel/bpf/btf.c

2000

return env->visit_states[type_id - env->btf->start_id] == RESOLVED;

kernel/bpf/btf.c

2003

static int env_stack_push(struct btf_verifier_env *env,

kernel/bpf/btf.c

2006

const struct btf *btf = env->btf;

kernel/bpf/btf.c

2009

if (env->top_stack == MAX_RESOLVE_DEPTH)

kernel/bpf/btf.c

2013

|| env->visit_states[type_id - btf->start_id] != NOT_VISITED)

kernel/bpf/btf.c

2016

env->visit_states[type_id - btf->start_id] = VISITED;

kernel/bpf/btf.c

2018

v = &env->stack[env->top_stack++];

kernel/bpf/btf.c

2023

if (env->resolve_mode == RESOLVE_TBD) {

kernel/bpf/btf.c

2025

env->resolve_mode = RESOLVE_PTR;

kernel/bpf/btf.c

2027

env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;

kernel/bpf/btf.c

2033

static void env_stack_set_next_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2036

env->stack[env->top_stack - 1].next_member = next_member;

kernel/bpf/btf.c

2039

static void env_stack_pop_resolved(struct btf_verifier_env *env,

kernel/bpf/btf.c

2043

u32 type_id = env->stack[--(env->top_stack)].type_id;

kernel/bpf/btf.c

2044

struct btf *btf = env->btf;

kernel/bpf/btf.c

2049

env->visit_states[type_id] = RESOLVED;

kernel/bpf/btf.c

2052

static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)

kernel/bpf/btf.c

2054

return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;

kernel/bpf/btf.c

2221

static int btf_df_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2226

btf_verifier_log_basic(env, struct_type,

kernel/bpf/btf.c

2231

static int btf_df_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2236

btf_verifier_log_basic(env, struct_type,

kernel/bpf/btf.c

2244

static int btf_generic_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2250

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2258

return btf_type_ops(member_type)->check_member(env, struct_type,

kernel/bpf/btf.c

2263

static int btf_df_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2266

btf_verifier_log_basic(env, v->t, "Unsupported resolve");

kernel/bpf/btf.c

2277

static int btf_int_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2289

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2300

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2307

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2315

static int btf_int_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2327

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2341

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2348

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2356

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2363

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2371

static s32 btf_int_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2379

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

2386

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2391

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

2397

btf_verifier_log_basic(env, t, "Invalid int_data:%x",

kernel/bpf/btf.c

2405

btf_verifier_log_type(env, t, "nr_bits exceeds %zu",

kernel/bpf/btf.c

2411

btf_verifier_log_type(env, t, "nr_bits exceeds type_size");

kernel/bpf/btf.c

2426

btf_verifier_log_type(env, t, "Unsupported encoding");

kernel/bpf/btf.c

2430

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2435

static void btf_int_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2440

btf_verifier_log(env,

kernel/bpf/btf.c

2636

static int btf_modifier_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2644

struct btf *btf = env->btf;

kernel/bpf/btf.c

2648

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2656

return btf_type_ops(resolved_type)->check_member(env, struct_type,

kernel/bpf/btf.c

2661

static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2669

struct btf *btf = env->btf;

kernel/bpf/btf.c

2673

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2681

return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,

kernel/bpf/btf.c

2686

static int btf_ptr_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2698

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2704

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2712

static int btf_ref_type_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2719

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2724

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

2729

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

2738

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

2739

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2743

value = btf_name_by_offset(env->btf, t->name_off);

kernel/bpf/btf.c

2745

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2750

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2755

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2760

static int btf_modifier_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2766

struct btf *btf = env->btf;

kernel/bpf/btf.c

2770

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2774

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2775

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2776

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2785

if (env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2792

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2797

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2802

static int btf_var_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2808

struct btf *btf = env->btf;

kernel/bpf/btf.c

2812

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2816

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2817

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2818

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2828

!env_type_is_resolve_sink(env, resolved_type) &&

kernel/bpf/btf.c

2829

!env_type_is_resolved(env, resolved_type_id))

kernel/bpf/btf.c

2830

return env_stack_push(env, resolved_type,

kernel/bpf/btf.c

2839

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2843

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2848

static int btf_ptr_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2854

struct btf *btf = env->btf;

kernel/bpf/btf.c

2858

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2862

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2863

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2864

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2882

!env_type_is_resolve_sink(env, resolved_type) &&

kernel/bpf/btf.c

2883

!env_type_is_resolved(env, resolved_type_id))

kernel/bpf/btf.c

2884

return env_stack_push(env, resolved_type,

kernel/bpf/btf.c

2889

if (env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2895

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2900

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2945

static void btf_ref_type_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2948

btf_verifier_log(env, "type_id=%u", t->type);

kernel/bpf/btf.c

2969

static s32 btf_fwd_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2974

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2979

btf_verifier_log_type(env, t, "type != 0");

kernel/bpf/btf.c

2985

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

2986

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2990

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2995

static void btf_fwd_type_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2998

btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");

kernel/bpf/btf.c

3010

static int btf_array_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

3018

struct btf *btf = env->btf;

kernel/bpf/btf.c

3021

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3031

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3039

static s32 btf_array_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

3047

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

3055

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

3060

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

3065

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

3070

btf_verifier_log_type(env, t, "size != 0");

kernel/bpf/btf.c

3078

btf_verifier_log_type(env, t, "Invalid elem");

kernel/bpf/btf.c

3083

btf_verifier_log_type(env, t, "Invalid index");

kernel/bpf/btf.c

3087

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

3092

static int btf_array_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

3098

struct btf *btf = env->btf;

kernel/bpf/btf.c

3106

btf_verifier_log_type(env, v->t, "Invalid index");

kernel/bpf/btf.c

3110

if (!env_type_is_resolve_sink(env, index_type) &&

kernel/bpf/btf.c

3111

!env_type_is_resolved(env, index_type_id))

kernel/bpf/btf.c

3112

return env_stack_push(env, index_type, index_type_id);

kernel/bpf/btf.c

3117

btf_verifier_log_type(env, v->t, "Invalid index");

kernel/bpf/btf.c

3126

btf_verifier_log_type(env, v->t,

kernel/bpf/btf.c

3131

if (!env_type_is_resolve_sink(env, elem_type) &&

kernel/bpf/btf.c

3132

!env_type_is_resolved(env, elem_type_id))

kernel/bpf/btf.c

3133

return env_stack_push(env, elem_type, elem_type_id);

kernel/bpf/btf.c

3137

btf_verifier_log_type(env, v->t, "Invalid elem");

kernel/bpf/btf.c

3142

btf_verifier_log_type(env, v->t, "Invalid array of int");

kernel/bpf/btf.c

3147

btf_verifier_log_type(env, v->t,

kernel/bpf/btf.c

3152

env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);

kernel/bpf/btf.c

3157

static void btf_array_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

3162

btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",

kernel/bpf/btf.c

3261

static int btf_struct_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

3270

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3278

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3286

static s32 btf_struct_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

3293

struct btf *btf = env->btf;

kernel/bpf/btf.c

3300

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

3308

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

3309

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

3313

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

3318

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3327

btf_verifier_log_member(env, t, member, "Invalid name");

kernel/bpf/btf.c

3332

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3339

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3349

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3355

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3360

btf_verifier_log_member(env, t, member, NULL);

kernel/bpf/btf.c

3367

static int btf_struct_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

3385

if (WARN_ON_ONCE(!env_type_is_resolved(env,

kernel/bpf/btf.c

3389

last_member_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

3392

err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,

kernel/bpf/btf.c

3396

err = btf_type_ops(last_member_type)->check_member(env, v->t,

kernel/bpf/btf.c

3405

const struct btf_type *member_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

3410

btf_verifier_log_member(env, v->t, member,

kernel/bpf/btf.c

3415

if (!env_type_is_resolve_sink(env, member_type) &&

kernel/bpf/btf.c

3416

!env_type_is_resolved(env, member_type_id)) {

kernel/bpf/btf.c

3417

env_stack_set_next_member(env, i + 1);

kernel/bpf/btf.c

3418

return env_stack_push(env, member_type, member_type_id);

kernel/bpf/btf.c

3422

err = btf_type_ops(member_type)->check_kflag_member(env, v->t,

kernel/bpf/btf.c

3426

err = btf_type_ops(member_type)->check_member(env, v->t,

kernel/bpf/btf.c

3433

env_stack_pop_resolved(env, 0, 0);

kernel/bpf/btf.c

3438

static void btf_struct_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

3441

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

4429

static int btf_enum_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

4438

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4446

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4454

static int btf_enum_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

4466

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

447

s32 (*check_meta)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4473

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4481

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4489

static s32 btf_enum_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4494

struct btf *btf = env->btf;

kernel/bpf/btf.c

450

int (*resolve)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4503

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4510

btf_verifier_log_type(env, t, "Unexpected size");

kernel/bpf/btf.c

4516

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4517

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

452

int (*check_member)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4521

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4525

btf_verifier_log(env, "\tInvalid name_offset:%u",

kernel/bpf/btf.c

4533

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4537

if (env->log.level == BPF_LOG_KERNEL)

kernel/bpf/btf.c

4540

btf_verifier_log(env, fmt_str,

kernel/bpf/btf.c

4548

static void btf_enum_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

4551

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

456

int (*check_kflag_member)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4597

static s32 btf_enum64_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

460

void (*log_details)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4602

struct btf *btf = env->btf;

kernel/bpf/btf.c

4611

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4618

btf_verifier_log_type(env, t, "Unexpected size");

kernel/bpf/btf.c

4624

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4625

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4629

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4633

btf_verifier_log(env, "\tInvalid name_offset:%u",

kernel/bpf/btf.c

4641

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4645

if (env->log.level == BPF_LOG_KERNEL)

kernel/bpf/btf.c

4649

btf_verifier_log(env, fmt_str,

kernel/bpf/btf.c

470

static int btf_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4700

static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4707

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4714

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4719

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4723

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4728

static void btf_func_proto_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

473

static int btf_func_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

4734

btf_verifier_log(env, "return=%u args=(", t->type);

kernel/bpf/btf.c

4736

btf_verifier_log(env, "void");

kernel/bpf/btf.c

4742

btf_verifier_log(env, "vararg");

kernel/bpf/btf.c

4746

btf_verifier_log(env, "%u %s", args[0].type,

kernel/bpf/btf.c

4747

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4750

btf_verifier_log(env, ", %u %s", args[i].type,

kernel/bpf/btf.c

4751

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4758

btf_verifier_log(env, ", %u %s", last_arg->type,

kernel/bpf/btf.c

4759

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4762

btf_verifier_log(env, ", vararg");

kernel/bpf/btf.c

4766

btf_verifier_log(env, ")");

kernel/bpf/btf.c

4787

static s32 btf_func_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4792

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4793

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4798

btf_verifier_log_type(env, t, "Invalid func linkage");

kernel/bpf/btf.c

4803

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4807

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4812

static int btf_func_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4819

err = btf_func_check(env, t);

kernel/bpf/btf.c

4823

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

4836

static s32 btf_var_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4844

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4851

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

4856

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4861

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4862

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4868

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

4875

btf_verifier_log_type(env, t, "Linkage not supported");

kernel/bpf/btf.c

4879

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4884

static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t)

kernel/bpf/btf.c

4888

btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);

kernel/bpf/btf.c

4900

static s32 btf_datasec_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4910

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4917

btf_verifier_log_type(env, t, "size == 0");

kernel/bpf/btf.c

4922

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4927

!btf_name_valid_section(env->btf, t->name_off)) {

kernel/bpf/btf.c

4928

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4932

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4937

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4943

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4949

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4956

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4961

btf_verifier_log_vsi(env, t, vsi, NULL);

kernel/bpf/btf.c

4966

btf_verifier_log_type(env, t, "Invalid btf_info size");

kernel/bpf/btf.c

4973

static int btf_datasec_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4977

struct btf *btf = env->btf;

kernel/bpf/btf.c

4980

env->resolve_mode = RESOLVE_TBD;

kernel/bpf/btf.c

4983

const struct btf_type *var_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

4986

btf_verifier_log_vsi(env, v->t, vsi,

kernel/bpf/btf.c

4991

if (!env_type_is_resolve_sink(env, var_type) &&

kernel/bpf/btf.c

4992

!env_type_is_resolved(env, var_type_id)) {

kernel/bpf/btf.c

4993

env_stack_set_next_member(env, i + 1);

kernel/bpf/btf.c

4994

return env_stack_push(env, var_type, var_type_id);

kernel/bpf/btf.c

4999

btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");

kernel/bpf/btf.c

5004

btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");

kernel/bpf/btf.c

5009

env_stack_pop_resolved(env, 0, 0);

kernel/bpf/btf.c

5013

static void btf_datasec_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

5016

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

5052

static s32 btf_float_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5057

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

5062

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

5068

btf_verifier_log_type(env, t, "Invalid type_size");

kernel/bpf/btf.c

5072

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

5077

static int btf_float_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

5096

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

5104

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

5112

static void btf_float_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

5115

btf_verifier_log(env, "size=%u", t->size);

kernel/bpf/btf.c

5127

static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5137

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

5143

value = btf_name_by_offset(env->btf, t->name_off);

kernel/bpf/btf.c

5145

btf_verifier_log_type(env, t, "Invalid value");

kernel/bpf/btf.c

5150

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

5156

btf_verifier_log_type(env, t, "Invalid component_idx");

kernel/bpf/btf.c

5160

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

5165

static int btf_decl_tag_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

5171

struct btf *btf = env->btf;

kernel/bpf/btf.c

5177

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

5181

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

5182

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

5183

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

5188

btf_verifier_log_type(env, v->t, "Invalid component_idx");

kernel/bpf/btf.c

5201

btf_verifier_log_type(env, v->t, "Invalid component_idx");

kernel/bpf/btf.c

5206

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

5211

static void btf_decl_tag_log(struct btf_verifier_env *env, const struct btf_type *t)

kernel/bpf/btf.c

5213

btf_verifier_log(env, "type=%u component_idx=%d", t->type,

kernel/bpf/btf.c

5226

static int btf_func_proto_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

5235

btf = env->btf;

kernel/bpf/btf.c

5245

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5250

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5255

!env_type_is_resolved(env, ret_type_id)) {

kernel/bpf/btf.c

5256

err = btf_resolve(env, ret_type, ret_type_id);

kernel/bpf/btf.c

5263

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5274

btf_verifier_log_type(env, t, "Invalid arg#%u",

kernel/bpf/btf.c

5288

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5293

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5300

btf_verifier_log_type(env, t,

kernel/bpf/btf.c

5306

!env_type_is_resolved(env, arg_type_id)) {

kernel/bpf/btf.c

5307

err = btf_resolve(env, arg_type, arg_type_id);

kernel/bpf/btf.c

5313

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5321

static int btf_func_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

5329

btf = env->btf;

kernel/bpf/btf.c

5333

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

5341

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5371

static s32 btf_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5379

btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",

kernel/bpf/btf.c

5380

env->log_type_id, meta_left, sizeof(*t));

kernel/bpf/btf.c

5387

btf_verifier_log(env, "[%u] Invalid kind:%u",

kernel/bpf/btf.c

5388

env->log_type_id, BTF_INFO_KIND(t->info));

kernel/bpf/btf.c

5392

if (!btf_name_offset_valid(env->btf, t->name_off)) {

kernel/bpf/btf.c

5393

btf_verifier_log(env, "[%u] Invalid name_offset:%u",

kernel/bpf/btf.c

5394

env->log_type_id, t->name_off);

kernel/bpf/btf.c

5398

var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);

kernel/bpf/btf.c

5407

static int btf_check_all_metas(struct btf_verifier_env *env)

kernel/bpf/btf.c

5409

struct btf *btf = env->btf;

kernel/bpf/btf.c

5417

env->log_type_id = btf->base_btf ? btf->start_id : 1;

kernel/bpf/btf.c

5422

meta_size = btf_check_meta(env, t, end - cur);

kernel/bpf/btf.c

5426

btf_add_type(env, t);

kernel/bpf/btf.c

5428

env->log_type_id++;

kernel/bpf/btf.c

5434

static bool btf_resolve_valid(struct btf_verifier_env *env,

kernel/bpf/btf.c

5438

struct btf *btf = env->btf;

kernel/bpf/btf.c

5440

if (!env_type_is_resolved(env, type_id))

kernel/bpf/btf.c

5475

static int btf_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

5478

u32 save_log_type_id = env->log_type_id;

kernel/bpf/btf.c

5482

env->resolve_mode = RESOLVE_TBD;

kernel/bpf/btf.c

5483

env_stack_push(env, t, type_id);

kernel/bpf/btf.c

5484

while (!err && (v = env_stack_peak(env))) {

kernel/bpf/btf.c

5485

env->log_type_id = v->type_id;

kernel/bpf/btf.c

5486

err = btf_type_ops(v->t)->resolve(env, v);

kernel/bpf/btf.c

5489

env->log_type_id = type_id;

kernel/bpf/btf.c

5491

btf_verifier_log_type(env, t,

kernel/bpf/btf.c

5495

btf_verifier_log_type(env, t, "Loop detected");

kernel/bpf/btf.c

5499

if (!err && !btf_resolve_valid(env, t, type_id)) {

kernel/bpf/btf.c

5500

btf_verifier_log_type(env, t, "Invalid resolve state");

kernel/bpf/btf.c

5504

env->log_type_id = save_log_type_id;

kernel/bpf/btf.c

5508

static int btf_check_all_types(struct btf_verifier_env *env)

kernel/bpf/btf.c

5510

struct btf *btf = env->btf;

kernel/bpf/btf.c

5515

err = env_resolve_init(env);

kernel/bpf/btf.c

5519

env->phase++;

kernel/bpf/btf.c

5524

env->log_type_id = type_id;

kernel/bpf/btf.c

5526

!env_type_is_resolved(env, type_id)) {

kernel/bpf/btf.c

5527

err = btf_resolve(env, t, type_id);

kernel/bpf/btf.c

5533

err = btf_func_proto_check(env, t);

kernel/bpf/btf.c

5542

static int btf_parse_type_sec(struct btf_verifier_env *env)

kernel/bpf/btf.c

5544

const struct btf_header *hdr = &env->btf->hdr;

kernel/bpf/btf.c

5549

btf_verifier_log(env, "Unaligned type_off");

kernel/bpf/btf.c

5553

if (!env->btf->base_btf && !hdr->type_len) {

kernel/bpf/btf.c

5554

btf_verifier_log(env, "No type found");

kernel/bpf/btf.c

5558

err = btf_check_all_metas(env);

kernel/bpf/btf.c

5562

return btf_check_all_types(env);

kernel/bpf/btf.c

5565

static int btf_parse_str_sec(struct btf_verifier_env *env)

kernel/bpf/btf.c

5568

struct btf *btf = env->btf;

kernel/bpf/btf.c

5577

btf_verifier_log(env, "String section is not at the end");

kernel/bpf/btf.c

5586

btf_verifier_log(env, "Invalid string section");

kernel/bpf/btf.c

5590

btf_verifier_log(env, "Invalid string section");

kernel/bpf/btf.c

5597

static int btf_parse_layout_sec(struct btf_verifier_env *env)

kernel/bpf/btf.c

5599

const struct btf_header *hdr = &env->btf->hdr;

kernel/bpf/btf.c

5600

struct btf *btf = env->btf;

kernel/bpf/btf.c

5609

btf_verifier_log(env, "Unaligned layout_off");

kernel/bpf/btf.c

5616

btf_verifier_log(env, "Layout section is too small");

kernel/bpf/btf.c

5620

btf_verifier_log(env, "layout_len is not multiple of %zu",

kernel/bpf/btf.c

5625

btf_verifier_log(env, "Layout section is too big");

kernel/bpf/btf.c

5647

static int btf_check_sec_info(struct btf_verifier_env *env,

kernel/bpf/btf.c

5656

btf = env->btf;

kernel/bpf/btf.c

5675

btf_verifier_log(env, "Invalid section offset");

kernel/bpf/btf.c

5680

btf_verifier_log(env, "Unsupported section found");

kernel/bpf/btf.c

5684

btf_verifier_log(env, "Section overlap found");

kernel/bpf/btf.c

5688

btf_verifier_log(env,

kernel/bpf/btf.c

5697

btf_verifier_log(env, "Unsupported section found");

kernel/bpf/btf.c

5704

static int btf_parse_hdr(struct btf_verifier_env *env)

kernel/bpf/btf.c

5710

btf = env->btf;

kernel/bpf/btf.c

5714

btf_verifier_log(env, "hdr_len not found");

kernel/bpf/btf.c

5721

btf_verifier_log(env, "btf_header not found");

kernel/bpf/btf.c

5732

btf_verifier_log(env, "Unsupported btf_header");

kernel/bpf/btf.c

5743

btf_verifier_log_hdr(env, btf_data_size);

kernel/bpf/btf.c

5746

btf_verifier_log(env, "Invalid magic");

kernel/bpf/btf.c

5751

btf_verifier_log(env, "Unsupported version");

kernel/bpf/btf.c

5756

btf_verifier_log(env, "Unsupported flags");

kernel/bpf/btf.c

5761

btf_verifier_log(env, "No data");

kernel/bpf/btf.c

5765

return btf_check_sec_info(env, btf_data_size);

kernel/bpf/btf.c

5910

static int btf_check_modifier_chain_length(struct btf_verifier_env *env,

kernel/bpf/btf.c

5931

btf_verifier_log(env, "Max chain length or cycle detected");

kernel/bpf/btf.c

5953

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

5961

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

5962

if (!env)

kernel/bpf/btf.c

5968

err = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);

kernel/bpf/btf.c

5977

env->btf = btf;

kernel/bpf/btf.c

5994

err = btf_parse_hdr(env);

kernel/bpf/btf.c

6000

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

6004

err = btf_parse_layout_sec(env);

kernel/bpf/btf.c

6008

err = btf_parse_type_sec(env);

kernel/bpf/btf.c

6012

err = btf_check_modifier_chain_length(env, btf, 1);

kernel/bpf/btf.c

6016

struct_meta_tab = btf_parse_struct_metas(&env->log, btf);

kernel/bpf/btf.c

6033

err = bpf_log_attr_finalize(attr_log, &env->log);

kernel/bpf/btf.c

6037

btf_verifier_env_free(env);

kernel/bpf/btf.c

6045

ret = bpf_log_attr_finalize(attr_log, &env->log);

kernel/bpf/btf.c

6049

btf_verifier_env_free(env);

kernel/bpf/btf.c

6384

static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name,

kernel/bpf/btf.c

6398

env->btf = btf;

kernel/bpf/btf.c

6406

err = btf_parse_hdr(env);

kernel/bpf/btf.c

6412

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

6416

err = btf_check_all_metas(env);

kernel/bpf/btf.c

6420

err = btf_check_modifier_chain_length(env, btf, 1);

kernel/bpf/btf.c

6439

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

6444

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

6445

if (!env)

kernel/bpf/btf.c

6448

log = &env->log;

kernel/bpf/btf.c

6450

btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF);

kernel/bpf/btf.c

6462

btf_verifier_env_free(env);

kernel/bpf/btf.c

6484

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

6494

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

6495

if (!env)

kernel/bpf/btf.c

6498

log = &env->log;

kernel/bpf/btf.c

6502

base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size);

kernel/bpf/btf.c

6516

env->btf = btf;

kernel/bpf/btf.c

6532

err = btf_parse_hdr(env);

kernel/bpf/btf.c

6538

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

6542

err = btf_check_all_metas(env);

kernel/bpf/btf.c

6546

err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf));

kernel/bpf/btf.c

6558

btf_verifier_env_free(env);

kernel/bpf/btf.c

6564

btf_verifier_env_free(env);

kernel/bpf/btf.c

7845

static int btf_scan_decl_tags(struct bpf_verifier_env *env,

kernel/bpf/btf.c

7876

bpf_log(&env->log,

kernel/bpf/btf.c

7895

bpf_log(&env->log, "arg#%d has unsupported set of tags\n", arg_idx);

kernel/bpf/btf.c

7900

bpf_log(&env->log, "arg#%d type tag fetching failure: %d\n", arg_idx, id);

kernel/bpf/btf.c

7907

static int btf_scan_type_tags(struct bpf_verifier_env *env,

kernel/bpf/btf.c

7932

bpf_log(&env->log,

kernel/bpf/btf.c

7942

static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf,

kernel/bpf/btf.c

7948

err = btf_scan_type_tags(env, btf, t->type, &tags);

kernel/bpf/btf.c

7977

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)

kernel/bpf/btf.c

7979

bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL;

kernel/bpf/btf.c

7980

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/btf.c

7981

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

7982

struct bpf_prog *prog = env->prog;

kernel/bpf/btf.c

7995

verifier_bug(env, "func_info undefined");

kernel/bpf/btf.c

8019

verifier_bug(env, "unreliable BTF for function %s()", tname);

kernel/bpf/btf.c

8053

err = btf_validate_return_type(env, btf, t, subprog);

kernel/bpf/btf.c

8069

err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags);

kernel/bpf/btf.c

8073

err = btf_scan_type_tags(env, btf, args[i].type, &tags);

kernel/bpf/cfg.c

102

static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)

kernel/bpf/cfg.c

104

int *insn_stack = env->cfg.insn_stack;

kernel/bpf/cfg.c

105

int *insn_state = env->cfg.insn_state;

kernel/bpf/cfg.c

113

if (w < 0 || w >= env->prog->len) {

kernel/bpf/cfg.c

114

verbose_linfo(env, t, "%d: ", t);

kernel/bpf/cfg.c

115

verbose(env, "jump out of range from insn %d to %d\n", t, w);

kernel/bpf/cfg.c

121

mark_prune_point(env, w);

kernel/bpf/cfg.c

122

mark_jmp_point(env, w);

kernel/bpf/cfg.c

129

if (env->cfg.cur_stack >= env->prog->len)

kernel/bpf/cfg.c

131

insn_stack[env->cfg.cur_stack++] = w;

kernel/bpf/cfg.c

134

if (env->bpf_capable)

kernel/bpf/cfg.c

136

verbose_linfo(env, t, "%d: ", t);

kernel/bpf/cfg.c

137

verbose_linfo(env, w, "%d: ", w);

kernel/bpf/cfg.c

138

verbose(env, "back-edge from insn %d to %d\n", t, w);

kernel/bpf/cfg.c

144

verifier_bug(env, "insn state internal bug");

kernel/bpf/cfg.c

151

struct bpf_verifier_env *env,

kernel/bpf/cfg.c

158

ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);

kernel/bpf/cfg.c

162

mark_prune_point(env, t + insn_sz);

kernel/bpf/cfg.c

164

mark_jmp_point(env, t + insn_sz);

kernel/bpf/cfg.c

168

mark_prune_point(env, t);

kernel/bpf/cfg.c

169

merge_callee_effects(env, t, w);

kernel/bpf/cfg.c

170

ret = push_insn(t, w, BRANCH, env);

kernel/bpf/cfg.c

279

static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,

kernel/bpf/cfg.c

287

for (i = 0; i < env->insn_array_map_cnt; i++) {

kernel/bpf/cfg.c

292

map = env->insn_array_maps[i];

kernel/bpf/cfg.c

318

verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);

kernel/bpf/cfg.c

327

create_jt(int t, struct bpf_verifier_env *env)

kernel/bpf/cfg.c

334

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/cfg.c

337

jt = jt_from_subprog(env, subprog_start, subprog_end);

kernel/bpf/cfg.c

344

verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",

kernel/bpf/cfg.c

355

static int visit_gotox_insn(int t, struct bpf_verifier_env *env)

kernel/bpf/cfg.c

357

int *insn_stack = env->cfg.insn_stack;

kernel/bpf/cfg.c

358

int *insn_state = env->cfg.insn_state;

kernel/bpf/cfg.c

363

jt = env->insn_aux_data[t].jt;

kernel/bpf/cfg.c

365

jt = create_jt(t, env);

kernel/bpf/cfg.c

369

env->insn_aux_data[t].jt = jt;

kernel/bpf/cfg.c

372

mark_prune_point(env, t);

kernel/bpf/cfg.c

375

if (w < 0 || w >= env->prog->len) {

kernel/bpf/cfg.c

376

verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);

kernel/bpf/cfg.c

380

mark_jmp_point(env, w);

kernel/bpf/cfg.c

386

if (env->cfg.cur_stack >= env->prog->len)

kernel/bpf/cfg.c

389

insn_stack[env->cfg.cur_stack++] = w;

kernel/bpf/cfg.c

402

static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t)

kernel/bpf/cfg.c

407

if (env->insn_aux_data[t].jt)

kernel/bpf/cfg.c

414

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/cfg.c

417

env->insn_aux_data[t].jt = jt;

kernel/bpf/cfg.c

426

static int visit_insn(int t, struct bpf_verifier_env *env)

kernel/bpf/cfg.c

428

struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];

kernel/bpf/cfg.c

432

return visit_func_call_insn(t, insns, env, true);

kernel/bpf/cfg.c

440

ret = visit_abnormal_return_insn(env, t);

kernel/bpf/cfg.c

445

return push_insn(t, t + insn_sz, FALLTHROUGH, env);

kernel/bpf/cfg.c

459

mark_prune_point(env, t);

kernel/bpf/cfg.c

470

mark_calls_callback(env, t);

kernel/bpf/cfg.c

471

mark_force_checkpoint(env, t);

kernel/bpf/cfg.c

472

mark_prune_point(env, t);

kernel/bpf/cfg.c

473

mark_jmp_point(env, t);

kernel/bpf/cfg.c

478

ret = bpf_get_helper_proto(env, insn->imm, &fp);

kernel/bpf/cfg.c

485

mark_subprog_might_sleep(env, t);

kernel/bpf/cfg.c

487

mark_subprog_changes_pkt_data(env, t);

kernel/bpf/cfg.c

489

ret = visit_abnormal_return_insn(env, t);

kernel/bpf/cfg.c

496

ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);

kernel/bpf/cfg.c

498

mark_prune_point(env, t);

kernel/bpf/cfg.c

51

static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)

kernel/bpf/cfg.c

510

mark_force_checkpoint(env, t);

kernel/bpf/cfg.c

518

mark_subprog_might_sleep(env, t);

kernel/bpf/cfg.c

520

mark_subprog_changes_pkt_data(env, t);

kernel/bpf/cfg.c

522

mark_subprog_might_throw(env, t);

kernel/bpf/cfg.c

524

return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);

kernel/bpf/cfg.c

528

return visit_gotox_insn(t, env);

kernel/bpf/cfg.c

536

ret = push_insn(t, t + off + 1, FALLTHROUGH, env);

kernel/bpf/cfg.c

540

mark_prune_point(env, t + off + 1);

kernel/bpf/cfg.c

541

mark_jmp_point(env, t + off + 1);

kernel/bpf/cfg.c

547

mark_prune_point(env, t);

kernel/bpf/cfg.c

549

mark_force_checkpoint(env, t);

kernel/bpf/cfg.c

55

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/cfg.c

551

ret = push_insn(t, t + 1, FALLTHROUGH, env);

kernel/bpf/cfg.c

555

return push_insn(t, t + insn->off + 1, BRANCH, env);

kernel/bpf/cfg.c

562

int bpf_check_cfg(struct bpf_verifier_env *env)

kernel/bpf/cfg.c

564

int insn_cnt = env->prog->len;

kernel/bpf/cfg.c

568

insn_state = env->cfg.insn_state = kvzalloc_objs(int, insn_cnt,

kernel/bpf/cfg.c

573

insn_stack = env->cfg.insn_stack = kvzalloc_objs(int, insn_cnt,

kernel/bpf/cfg.c

580

ex_insn_beg = env->exception_callback_subprog

kernel/bpf/cfg.c

581

? env->subprog_info[env->exception_callback_subprog].start

kernel/bpf/cfg.c

586

env->cfg.cur_stack = 1;

kernel/bpf/cfg.c

589

while (env->cfg.cur_stack > 0) {

kernel/bpf/cfg.c

59

static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)

kernel/bpf/cfg.c

590

int t = insn_stack[env->cfg.cur_stack - 1];

kernel/bpf/cfg.c

592

ret = visit_insn(t, env);

kernel/bpf/cfg.c

596

env->cfg.cur_stack--;

kernel/bpf/cfg.c

602

verifier_bug(env, "visit_insn internal bug");

kernel/bpf/cfg.c

609

if (env->cfg.cur_stack < 0) {

kernel/bpf/cfg.c

610

verifier_bug(env, "pop stack internal bug");

kernel/bpf/cfg.c

618

env->cfg.cur_stack = 1;

kernel/bpf/cfg.c

623

struct bpf_insn *insn = &env->prog->insnsi[i];

kernel/bpf/cfg.c

626

verbose(env, "unreachable insn %d\n", i);

kernel/bpf/cfg.c

63

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/cfg.c

632

verbose(env, "jump into the middle of ldimm64 insn %d\n", i);

kernel/bpf/cfg.c

640

env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;

kernel/bpf/cfg.c

641

env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;

kernel/bpf/cfg.c

646

env->cfg.insn_state = env->cfg.insn_stack = NULL;

kernel/bpf/cfg.c

655

int bpf_compute_postorder(struct bpf_verifier_env *env)

kernel/bpf/cfg.c

661

postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/cfg.c

662

state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/cfg.c

663

stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/cfg.c

67

static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off)

kernel/bpf/cfg.c

671

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/cfg.c

672

env->subprog_info[i].postorder_start = cur_postorder;

kernel/bpf/cfg.c

673

stack[0] = env->subprog_info[i].start;

kernel/bpf/cfg.c

683

succ = bpf_insn_successors(env, top);

kernel/bpf/cfg.c

693

env->subprog_info[i].postorder_start = cur_postorder;

kernel/bpf/cfg.c

694

env->cfg.insn_postorder = postorder;

kernel/bpf/cfg.c

695

env->cfg.cur_postorder = cur_postorder;

kernel/bpf/cfg.c

708

int bpf_compute_scc(struct bpf_verifier_env *env)

kernel/bpf/cfg.c

71

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/cfg.c

712

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/cfg.c

713

const u32 insn_cnt = env->prog->len;

kernel/bpf/cfg.c

8

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/cfg.c

81

static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)

kernel/bpf/cfg.c

827

succ = bpf_insn_successors(env, w);

kernel/bpf/cfg.c

85

caller = bpf_find_containing_subprog(env, t);

kernel/bpf/cfg.c

856

if (bpf_calls_callback(env, w)) /* implicit loop? */

kernel/bpf/cfg.c

86

callee = bpf_find_containing_subprog(env, w);

kernel/bpf/cfg.c

870

env->scc_info = kvzalloc_objs(*env->scc_info, next_scc_id,

kernel/bpf/cfg.c

872

if (!env->scc_info) {

kernel/bpf/cfg.c

876

env->scc_cnt = next_scc_id;

kernel/bpf/check_btf.c

10

static int check_abnormal_return(struct bpf_verifier_env *env)

kernel/bpf/check_btf.c

101

verbose(env,

kernel/bpf/check_btf.c

110

verbose(env, "invalid type id %d in func info",

kernel/bpf/check_btf.c

133

static int check_btf_func(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

14

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/check_btf.c

149

if (check_abnormal_return(env))

kernel/bpf/check_btf.c

15

if (env->subprog_info[i].has_ld_abs) {

kernel/bpf/check_btf.c

153

if (nfuncs != env->subprog_cnt) {

kernel/bpf/check_btf.c

154

verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");

kernel/bpf/check_btf.c

16

verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");

kernel/bpf/check_btf.c

160

prog = env->prog;

kernel/bpf/check_btf.c

175

if (env->subprog_info[i].start != krecord[i].insn_off) {

kernel/bpf/check_btf.c

176

verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");

kernel/bpf/check_btf.c

189

if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {

kernel/bpf/check_btf.c

19

if (env->subprog_info[i].has_tail_call) {

kernel/bpf/check_btf.c

190

verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");

kernel/bpf/check_btf.c

193

if (i && !scalar_return && env->subprog_info[i].has_tail_call) {

kernel/bpf/check_btf.c

194

verbose(env, "tail_call is only allowed in functions that return 'int'.\n");

kernel/bpf/check_btf.c

198

env->subprog_info[i].name = btf_name_by_offset(btf, type->name_off);

kernel/bpf/check_btf.c

20

verbose(env, "tail_call is not allowed in subprogs without BTF\n");

kernel/bpf/check_btf.c

213

static int check_btf_line(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

245

prog = env->prog;

kernel/bpf/check_btf.c

249

sub = env->subprog_info;

kernel/bpf/check_btf.c

257

verbose(env, "nonzero tailing record in line_info");

kernel/bpf/check_btf.c

284

verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",

kernel/bpf/check_btf.c

292

verbose(env,

kernel/bpf/check_btf.c

301

verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);

kernel/bpf/check_btf.c

306

if (s != env->subprog_cnt) {

kernel/bpf/check_btf.c

31

static int check_btf_func_early(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

311

verbose(env, "missing bpf_line_info for func#%u\n", s);

kernel/bpf/check_btf.c

321

if (s != env->subprog_cnt) {

kernel/bpf/check_btf.c

322

verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",

kernel/bpf/check_btf.c

323

env->subprog_cnt - s, s);

kernel/bpf/check_btf.c

341

static int check_core_relo(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

347

struct bpf_prog *prog = env->prog;

kernel/bpf/check_btf.c

350

.log = &env->log,

kernel/bpf/check_btf.c

380

verbose(env, "nonzero tailing record in core_relo");

kernel/bpf/check_btf.c

395

verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",

kernel/bpf/check_btf.c

410

int bpf_check_btf_info_early(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

418

if (check_abnormal_return(env))

kernel/bpf/check_btf.c

430

env->prog->aux->btf = btf;

kernel/bpf/check_btf.c

432

err = check_btf_func_early(env, attr, uattr);

kernel/bpf/check_btf.c

438

int bpf_check_btf_info(struct bpf_verifier_env *env,

kernel/bpf/check_btf.c

445

if (check_abnormal_return(env))

kernel/bpf/check_btf.c

450

err = check_btf_func(env, attr, uattr);

kernel/bpf/check_btf.c

454

err = check_btf_line(env, attr, uattr);

kernel/bpf/check_btf.c

458

err = check_core_relo(env, attr, uattr);

kernel/bpf/check_btf.c

47

if (check_abnormal_return(env))

kernel/bpf/check_btf.c

56

verbose(env, "invalid func info rec size %u\n", urec_size);

kernel/bpf/check_btf.c

60

prog = env->prog;

kernel/bpf/check_btf.c

74

verbose(env, "nonzero tailing record in func info");

kernel/bpf/check_btf.c

8

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/check_btf.c

95

verbose(env,

kernel/bpf/const_fold.c

149

int subprog = bpf_find_subprog(env, idx + insn->imm + 1);

kernel/bpf/const_fold.c

160

dst->map_index = env->insn_aux_data[idx].map_index;

kernel/bpf/const_fold.c

161

dst->val = env->insn_aux_data[idx].map_off;

kernel/bpf/const_fold.c

165

dst->map_index = env->insn_aux_data[idx].map_index;

kernel/bpf/const_fold.c

178

struct bpf_map *map = env->used_maps[src->map_index];

kernel/bpf/const_fold.c

238

int bpf_compute_const_regs(struct bpf_verifier_env *env)

kernel/bpf/const_fold.c

241

struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;

kernel/bpf/const_fold.c

242

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/const_fold.c

243

int insn_cnt = env->prog->len;

kernel/bpf/const_fold.c

256

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/const_fold.c

257

int start = env->subprog_info[i].start;

kernel/bpf/const_fold.c

265

for (i = env->cfg.cur_postorder - 1; i >= 0; i--) {

kernel/bpf/const_fold.c

266

int idx = env->cfg.insn_postorder[i];

kernel/bpf/const_fold.c

272

const_reg_xfer(env, ci_out, insn, insns, idx);

kernel/bpf/const_fold.c

274

succ = bpf_insn_successors(env, idx);

kernel/bpf/const_fold.c

345

int bpf_prune_dead_branches(struct bpf_verifier_env *env)

kernel/bpf/const_fold.c

347

struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;

kernel/bpf/const_fold.c

348

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/const_fold.c

349

int insn_cnt = env->prog->len;

kernel/bpf/const_fold.c

391

bpf_log(&env->log, "Unknown conditional jump %x\n", insn->code);

kernel/bpf/const_fold.c

401

kvfree(env->cfg.insn_postorder);

kernel/bpf/const_fold.c

402

env->cfg.insn_postorder = NULL;

kernel/bpf/const_fold.c

403

return bpf_compute_postorder(env);

kernel/bpf/const_fold.c

50

static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *ci_out,

kernel/bpf/core.c

1554

struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)

kernel/bpf/core.c

1562

if (WARN_ON_ONCE(env && env->prog != prog))

kernel/bpf/core.c

1570

if (env)

kernel/bpf/core.c

1571

env->prog = clone;

kernel/bpf/core.c

1600

if (env)

kernel/bpf/core.c

1601

tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);

kernel/bpf/core.c

1606

if (env)

kernel/bpf/core.c

1608

env->prog = prog;

kernel/bpf/core.c

1620

if (env)

kernel/bpf/core.c

1621

env->prog = clone;

kernel/bpf/core.c

1633

bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,

kernel/bpf/core.c

1636

if (!env)

kernel/bpf/core.c

1639

return env->insn_aux_data[insn_idx].indirect_target;

kernel/bpf/core.c

1642

u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog)

kernel/bpf/core.c

1646

if (!env)

kernel/bpf/core.c

1648

sub = &env->subprog_info[prog->aux->func_idx];

kernel/bpf/core.c

2620

static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

kernel/bpf/core.c

2627

return bpf_int_jit_compile(env, prog);

kernel/bpf/core.c

2629

if (env) {

kernel/bpf/core.c

2635

orig_insn_aux = bpf_dup_insn_aux_data(env);

kernel/bpf/core.c

2641

prog = bpf_jit_blind_constants(env, prog);

kernel/bpf/core.c

2649

prog = bpf_int_jit_compile(env, prog);

kernel/bpf/core.c

2652

if (env)

kernel/bpf/core.c

2661

if (env)

kernel/bpf/core.c

2662

bpf_restore_insn_aux_data(env, orig_insn_aux);

kernel/bpf/core.c

2667

struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,

kernel/bpf/core.c

2696

fp = bpf_prog_jit_compile(env, fp);

kernel/bpf/core.c

3225

struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)

kernel/bpf/fixups.c

1000

static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)

kernel/bpf/fixups.c

1002

for (int i = 0; i < env->subprog_cnt; i++)

kernel/bpf/fixups.c

1003

env->subprog_info[i].start = orig_starts[i];

kernel/bpf/fixups.c

1005

env->subprog_info[env->subprog_cnt].start = env->prog->len;

kernel/bpf/fixups.c

1008

struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1013

size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);

kernel/bpf/fixups.c

1016

memcpy(new_aux, env->insn_aux_data, size);

kernel/bpf/fixups.c

1020

void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

1024

vfree(env->insn_aux_data);

kernel/bpf/fixups.c

1025

env->insn_aux_data = orig_insn_aux;

kernel/bpf/fixups.c

1028

static int jit_subprogs(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1030

struct bpf_prog *prog = env->prog, **func, *tmp;

kernel/bpf/fixups.c

1045

subprog = bpf_find_subprog(env, i + insn->imm + 1);

kernel/bpf/fixups.c

1046

if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",

kernel/bpf/fixups.c

1056

env->insn_aux_data[i].call_imm = insn->imm;

kernel/bpf/fixups.c

107

static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1079

func = kzalloc_objs(prog, env->subprog_cnt);

kernel/bpf/fixups.c

1083

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1085

subprog_end = env->subprog_info[i + 1].start;

kernel/bpf/fixups.c

112

tab = env->prog->aux->kfunc_tab;

kernel/bpf/fixups.c

1125

func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;

kernel/bpf/fixups.c

1126

if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)

kernel/bpf/fixups.c

1136

func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;

kernel/bpf/fixups.c

1138

func[i]->aux->used_maps = env->used_maps;

kernel/bpf/fixups.c

1139

func[i]->aux->used_map_cnt = env->used_map_cnt;

kernel/bpf/fixups.c

1158

func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;

kernel/bpf/fixups.c

1159

func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;

kernel/bpf/fixups.c

1160

func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;

kernel/bpf/fixups.c

1161

func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;

kernel/bpf/fixups.c

1164

func[i]->aux->exception_boundary = env->seen_exception;

kernel/bpf/fixups.c

1165

func[i] = bpf_int_jit_compile(env, func[i]);

kernel/bpf/fixups.c

117

err = set_kfunc_desc_imm(env, &tab->descs[i]);

kernel/bpf/fixups.c

1177

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1204

func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;

kernel/bpf/fixups.c

1205

func[i]->aux->real_func_cnt = env->subprog_cnt;

kernel/bpf/fixups.c

1207

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1209

tmp = bpf_int_jit_compile(env, func[i]);

kernel/bpf/fixups.c

1211

verbose(env, "JIT doesn't support bpf-to-bpf calls\n");

kernel/bpf/fixups.c

1222

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1231

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1237

for (i = 1; i < env->subprog_cnt; i++)

kernel/bpf/fixups.c

1246

insn[0].imm = env->insn_aux_data[i].call_imm;

kernel/bpf/fixups.c

1253

insn->imm = env->insn_aux_data[i].call_imm;

kernel/bpf/fixups.c

1254

subprog = bpf_find_subprog(env, i + insn->imm + 1);

kernel/bpf/fixups.c

1264

prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;

kernel/bpf/fixups.c

1265

prog->aux->real_func_cnt = env->subprog_cnt;

kernel/bpf/fixups.c

1266

prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;

kernel/bpf/fixups.c

127

static int add_kfunc_in_insns(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

1284

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1296

int bpf_jit_subprogs(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1305

if (env->subprog_cnt <= 1)

kernel/bpf/fixups.c

1308

prog = orig_prog = env->prog;

kernel/bpf/fixups.c

1310

orig_insn_aux = bpf_dup_insn_aux_data(env);

kernel/bpf/fixups.c

1315

orig_subprog_starts = bpf_dup_subprog_starts(env);

kernel/bpf/fixups.c

1321

prog = bpf_jit_blind_constants(env, prog);

kernel/bpf/fixups.c

1330

err = jit_subprogs(env);

kernel/bpf/fixups.c

134

ret = bpf_add_kfunc_call(env, insn->imm, insn->off);

kernel/bpf/fixups.c

1346

prog = env->prog = orig_prog;

kernel/bpf/fixups.c

1359

insn->imm = env->insn_aux_data[i].call_imm;

kernel/bpf/fixups.c

1366

bpf_restore_subprog_starts(env, orig_subprog_starts);

kernel/bpf/fixups.c

1367

bpf_restore_insn_aux_data(env, orig_insn_aux);

kernel/bpf/fixups.c

1376

int bpf_fixup_call_args(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1379

struct bpf_prog *prog = env->prog;

kernel/bpf/fixups.c

1386

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1387

struct bpf_subprog_info *subprog = &env->subprog_info[i];

kernel/bpf/fixups.c

1391

verbose(env,

kernel/bpf/fixups.c

1398

if (env->prog->jit_requested &&

kernel/bpf/fixups.c

1399

!bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/fixups.c

14

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/fixups.c

1400

err = bpf_jit_subprogs(env);

kernel/bpf/fixups.c

1408

verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");

kernel/bpf/fixups.c

1411

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

1412

if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) {

kernel/bpf/fixups.c

1413

verbose(env, "stack args are not supported in non-JITed programs\n");

kernel/bpf/fixups.c

1417

if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {

kernel/bpf/fixups.c

1421

verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");

kernel/bpf/fixups.c

1429

verbose(env, "callbacks are not allowed in non-JITed programs\n");

kernel/bpf/fixups.c

143

static int get_callee_stack_depth(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

1435

depth = get_callee_stack_depth(env, insn, i);

kernel/bpf/fixups.c

1440

verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",

kernel/bpf/fixups.c

1452

static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)

kernel/bpf/fixups.c

1454

struct bpf_subprog_info *info = env->subprog_info;

kernel/bpf/fixups.c

1455

int cnt = env->subprog_cnt;

kernel/bpf/fixups.c

1459

if (env->hidden_subprog_cnt) {

kernel/bpf/fixups.c

1460

verifier_bug(env, "only one hidden subprog supported");

kernel/bpf/fixups.c

1467

prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);

kernel/bpf/fixups.c

1470

env->prog = prog;

kernel/bpf/fixups.c

1473

env->subprog_cnt++;

kernel/bpf/fixups.c

1474

env->hidden_subprog_cnt++;

kernel/bpf/fixups.c

148

subprog = bpf_find_subprog(env, start);

kernel/bpf/fixups.c

1481

int bpf_do_misc_fixups(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

1483

struct bpf_prog *prog = env->prog;

kernel/bpf/fixups.c

149

if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))

kernel/bpf/fixups.c

1491

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/fixups.c

1495

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/fixups.c

1499

if (env->seen_exception && !env->exception_callback_subprog) {

kernel/bpf/fixups.c

1502

*patch++ = env->prog->insnsi[insn_cnt - 1];

kernel/bpf/fixups.c

1505

ret = add_hidden_subprog(env, insn_buf, patch - insn_buf);

kernel/bpf/fixups.c

1508

prog = env->prog;

kernel/bpf/fixups.c

151

return env->subprog_info[subprog].stack_depth;

kernel/bpf/fixups.c

1511

env->exception_callback_subprog = env->subprog_cnt - 1;

kernel/bpf/fixups.c

1513

bpf_mark_subprog_exc_cb(env, env->exception_callback_subprog);

kernel/bpf/fixups.c

1519

(((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {

kernel/bpf/fixups.c

1529

if (env->insn_aux_data[i + delta].needs_zext)

kernel/bpf/fixups.c

1552

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1557

env->prog = prog = new_prog;

kernel/bpf/fixups.c

159

static void adjust_insn_aux_data(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

162

struct bpf_insn_aux_data *data = env->insn_aux_data;

kernel/bpf/fixups.c

1643

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1648

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1673

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1678

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1687

cnt = env->ops->gen_ld_abs(insn, insn_buf);

kernel/bpf/fixups.c

1689

verifier_bug(env, "%d insns generated for ld_abs", cnt);

kernel/bpf/fixups.c

1693

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1698

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1712

aux = &env->insn_aux_data[i + delta];

kernel/bpf/fixups.c

1746

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1751

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1791

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1796

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1812

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1817

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1827

ret = bpf_fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);

kernel/bpf/fixups.c

1833

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1838

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1860

if (!bpf_allow_tail_call_in_subprogs(env))

kernel/bpf/fixups.c

1872

aux = &env->insn_aux_data[i + delta];

kernel/bpf/fixups.c

1873

if (env->bpf_capable && !prog->blinding_requested &&

kernel/bpf/fixups.c

1887

verbose(env, "adding tail call poke descriptor failed\n");

kernel/bpf/fixups.c

1905

verbose(env, "tail_call abusing map_ptr\n");

kernel/bpf/fixups.c

1918

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1923

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1951

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1956

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1962

if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {

kernel/bpf/fixups.c

1970

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

1975

env->prog = prog = new_prog;

kernel/bpf/fixups.c

1994

aux = &env->insn_aux_data[i + delta];

kernel/bpf/fixups.c

200

static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/fixups.c

2006

verifier_bug(env, "%d insns generated for map lookup", cnt);

kernel/bpf/fixups.c

2010

new_prog = bpf_patch_insn_data(env, i + delta,

kernel/bpf/fixups.c

2016

env->prog = prog = new_prog;

kernel/bpf/fixups.c

207

for (i = 0; i <= env->subprog_cnt; i++) {

kernel/bpf/fixups.c

208

if (env->subprog_info[i].start <= off)

kernel/bpf/fixups.c

2093

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,

kernel/bpf/fixups.c

2099

env->prog = prog = new_prog;

kernel/bpf/fixups.c

210

env->subprog_info[i].start += len - 1;

kernel/bpf/fixups.c

2107

bpf_verifier_inlines_helper_call(env, insn->imm)) {

kernel/bpf/fixups.c

2122

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2127

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2134

bpf_verifier_inlines_helper_call(env, insn->imm)) {

kernel/bpf/fixups.c

214

static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/fixups.c

2140

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2145

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2174

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2179

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2206

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

221

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/fixups.c

2211

env->prog = prog = new_prog;

kernel/bpf/fixups.c

222

bpf_insn_array_adjust(env->insn_array_maps[i], off, len);

kernel/bpf/fixups.c

2232

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2237

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2248

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);

kernel/bpf/fixups.c

225

static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/fixups.c

2252

env->prog = prog = new_prog;

kernel/bpf/fixups.c

229

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/fixups.c

230

bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);

kernel/bpf/fixups.c

2303

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2308

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2321

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

2326

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2331

fn = env->ops->get_func_proto(insn->imm, env->prog);

kernel/bpf/fixups.c

2336

verifier_bug(env,

kernel/bpf/fixups.c

2349

verbose(env, "stack size %d(extra %d) is too large\n",

kernel/bpf/fixups.c

2361

env->prog->aux->stack_depth = subprogs[0].stack_depth;

kernel/bpf/fixups.c

2362

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/fixups.c

2372

verifier_bug(env, "stack_slots supports may_goto only");

kernel/bpf/fixups.c

2387

insn_buf[cnt++] = env->prog->insnsi[subprog_start];

kernel/bpf/fixups.c

2389

new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);

kernel/bpf/fixups.c

2392

env->prog = prog = new_prog;

kernel/bpf/fixups.c

2399

WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));

kernel/bpf/fixups.c

2408

verifier_bug(env, "poke tab is misconfigured");

kernel/bpf/fixups.c

2414

verbose(env, "tracking tail call prog failed\n");

kernel/bpf/fixups.c

2419

ret = sort_kfunc_descs_by_imm_off(env);

kernel/bpf/fixups.c

2426

static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

2439

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/fixups.c

247

struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,

kernel/bpf/fixups.c

2488

new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);

kernel/bpf/fixups.c

2493

callback_start = env->subprog_info[callback_subprogno].start;

kernel/bpf/fixups.c

2518

int bpf_optimize_bpf_loop(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

2520

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/fixups.c

2522

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/fixups.c

2523

int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

2530

&env->insn_aux_data[i + delta].loop_inline_state;

kernel/bpf/fixups.c

2536

new_prog = inline_bpf_loop(env,

kernel/bpf/fixups.c

254

new_data = vrealloc(env->insn_aux_data,

kernel/bpf/fixups.c

2545

env->prog = new_prog;

kernel/bpf/fixups.c

255

array_size(env->prog->len + len - 1,

kernel/bpf/fixups.c

2558

env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;

kernel/bpf/fixups.c

2566

int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

2568

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/fixups.c

2569

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/fixups.c

2570

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/fixups.c

2571

int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

261

env->insn_aux_data = new_data;

kernel/bpf/fixups.c

264

new_prog = bpf_patch_insn_single(env->prog, off, patch, len);

kernel/bpf/fixups.c

267

verbose(env,

kernel/bpf/fixups.c

269

env->insn_aux_data[off].orig_idx);

kernel/bpf/fixups.c

272

adjust_insn_aux_data(env, new_prog, off, len);

kernel/bpf/fixups.c

273

adjust_subprog_starts(env, off, len);

kernel/bpf/fixups.c

274

adjust_insn_arrays(env, off, len);

kernel/bpf/fixups.c

317

static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

323

for (i = 0; i < env->subprog_cnt; i++)

kernel/bpf/fixups.c

324

if (env->subprog_info[i].start >= off)

kernel/bpf/fixups.c

327

for (j = i; j < env->subprog_cnt; j++)

kernel/bpf/fixups.c

328

if (env->subprog_info[j].start >= off + cnt)

kernel/bpf/fixups.c

333

if (env->subprog_info[j].start != off + cnt)

kernel/bpf/fixups.c

337

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/fixups.c

341

move = env->subprog_cnt + 1 - j;

kernel/bpf/fixups.c

343

memmove(env->subprog_info + i,

kernel/bpf/fixups.c

344

env->subprog_info + j,

kernel/bpf/fixups.c

345

sizeof(*env->subprog_info) * move);

kernel/bpf/fixups.c

346

env->subprog_cnt -= j - i;

kernel/bpf/fixups.c

362

if (env->subprog_info[i].start == off)

kernel/bpf/fixups.c

367

for (; i <= env->subprog_cnt; i++)

kernel/bpf/fixups.c

368

env->subprog_info[i].start -= cnt;

kernel/bpf/fixups.c

373

static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,

kernel/bpf/fixups.c

376

struct bpf_prog *prog = env->prog;

kernel/bpf/fixups.c

423

for (i = 0; i <= env->subprog_cnt; i++)

kernel/bpf/fixups.c

424

if (env->subprog_info[i].linfo_idx > l_off) {

kernel/bpf/fixups.c

428

if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)

kernel/bpf/fixups.c

429

env->subprog_info[i].linfo_idx -= l_cnt;

kernel/bpf/fixups.c

431

env->subprog_info[i].linfo_idx = l_off;

kernel/bpf/fixups.c

440

void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)

kernel/bpf/fixups.c

442

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/fixups.c

443

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/fixups.c

458

static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

kernel/bpf/fixups.c

460

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/fixups.c

461

unsigned int orig_prog_len = env->prog->len;

kernel/bpf/fixups.c

464

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/fixups.c

465

bpf_prog_offload_remove_insns(env, off, cnt);

kernel/bpf/fixups.c

468

bpf_clear_insn_aux_data(env, off, cnt);

kernel/bpf/fixups.c

470

err = bpf_remove_insns(env->prog, off, cnt);

kernel/bpf/fixups.c

474

err = adjust_subprog_starts_after_remove(env, off, cnt);

kernel/bpf/fixups.c

478

err = bpf_adj_linfo_after_remove(env, off, cnt);

kernel/bpf/fixups.c

482

adjust_insn_arrays_after_remove(env, off, cnt);

kernel/bpf/fixups.c

507

void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

509

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/fixups.c

511

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/fixups.c

512

const int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

526

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/fixups.c

527

bpf_prog_offload_replace_insn(env, i, &ja);

kernel/bpf/fixups.c

533

int bpf_opt_remove_dead_code(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

535

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/fixups.c

536

int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

548

err = verifier_remove_insns(env, i, j);

kernel/bpf/fixups.c

551

insn_cnt = env->prog->len;

kernel/bpf/fixups.c

557

int bpf_opt_remove_nops(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

559

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/fixups.c

560

int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

571

err = verifier_remove_insns(env, i, 1);

kernel/bpf/fixups.c

582

int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,

kernel/bpf/fixups.c

587

struct bpf_insn *zext_patch = env->insn_buf;

kernel/bpf/fixups.c

588

struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2];

kernel/bpf/fixups.c

589

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/fixups.c

590

int i, patch_len, delta = 0, len = env->prog->len;

kernel/bpf/fixups.c

591

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/fixups.c

661

if (verifier_bug_if(load_reg == -1, env,

kernel/bpf/fixups.c

671

new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);

kernel/bpf/fixups.c

674

env->prog = new_prog;

kernel/bpf/fixups.c

676

aux = env->insn_aux_data;

kernel/bpf/fixups.c

688

int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

690

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/fixups.c

691

const struct bpf_verifier_ops *ops = env->ops;

kernel/bpf/fixups.c

693

const int insn_cnt = env->prog->len;

kernel/bpf/fixups.c

694

struct bpf_insn *epilogue_buf = env->epilogue_buf;

kernel/bpf/fixups.c

695

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/fixups.c

704

epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,

kernel/bpf/fixups.c

707

verifier_bug(env, "epilogue is too long");

kernel/bpf/fixups.c

715

insn_buf[cnt++] = env->prog->insnsi[0];

kernel/bpf/fixups.c

716

new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);

kernel/bpf/fixups.c

719

env->prog = new_prog;

kernel/bpf/fixups.c

722

ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);

kernel/bpf/fixups.c

728

if (ops->gen_prologue || env->seen_direct_write) {

kernel/bpf/fixups.c

730

verifier_bug(env, "gen_prologue is null");

kernel/bpf/fixups.c

733

cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,

kernel/bpf/fixups.c

734

env->prog);

kernel/bpf/fixups.c

736

verifier_bug(env, "prologue is too long");

kernel/bpf/fixups.c

739

new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);

kernel/bpf/fixups.c

743

env->prog = new_prog;

kernel/bpf/fixups.c

746

ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);

kernel/bpf/fixups.c

753

WARN_ON(adjust_jmp_off(env->prog, 0, delta));

kernel/bpf/fixups.c

755

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/fixups.c

758

insn = env->prog->insnsi + delta;

kernel/bpf/fixups.c

764

if (env->insn_aux_data[i + delta].nospec) {

kernel/bpf/fixups.c

765

WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);

kernel/bpf/fixups.c

771

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

776

env->prog = new_prog;

kernel/bpf/fixups.c

807

env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {

kernel/bpf/fixups.c

809

env->prog->aux->num_exentries++;

kernel/bpf/fixups.c

835

env->insn_aux_data[i + delta].nospec_result) {

kernel/bpf/fixups.c

844

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

849

env->prog = new_prog;

kernel/bpf/fixups.c

854

switch ((int)env->insn_aux_data[i + delta].ptr_type) {

kernel/bpf/fixups.c

88

static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)

kernel/bpf/fixups.c

887

env->prog->aux->num_exentries++;

kernel/bpf/fixups.c

893

verbose(env, "sign extending loads from arena are not supported yet\n");

kernel/bpf/fixups.c

900

env->prog->aux->num_exentries++;

kernel/bpf/fixups.c

906

ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;

kernel/bpf/fixups.c

922

verifier_bug(env, "narrow ctx access misconfigured");

kernel/bpf/fixups.c

937

cnt = convert_ctx_access(type, insn, insn_buf, env->prog,

kernel/bpf/fixups.c

941

verifier_bug(env, "error during ctx access conversion (%d)", cnt);

kernel/bpf/fixups.c

949

verifier_bug(env, "narrow ctx load misconfigured");

kernel/bpf/fixups.c

974

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/fixups.c

98

verbose(env, "address of kernel func_id %u is out of range\n",

kernel/bpf/fixups.c

981

env->prog = new_prog;

kernel/bpf/fixups.c

988

static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)

kernel/bpf/fixups.c

992

starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);

kernel/bpf/fixups.c

994

for (int i = 0; i < env->subprog_cnt; i++)

kernel/bpf/fixups.c

995

starts[i] = env->subprog_info[i].start;

kernel/bpf/liveness.c

101

static struct func_instance *lookup_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1031

static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx,

kernel/bpf/liveness.c

1038

if (!(env->log.level & BPF_LOG_LEVEL2))

kernel/bpf/liveness.c

1044

verbose(env, "%3d: ", idx);

kernel/bpf/liveness.c

1045

bpf_verbose_insn(env, insn);

kernel/bpf/liveness.c

1046

bpf_vlog_reset(&env->log, env->log.end_pos - 1);

kernel/bpf/liveness.c

1049

verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]);

kernel/bpf/liveness.c

1050

verbose(env, " -> "); verbose_arg_track(env, &at_out[i]);

kernel/bpf/liveness.c

1059

verbose(env, "%3d: ", idx);

kernel/bpf/liveness.c

1060

bpf_verbose_insn(env, insn);

kernel/bpf/liveness.c

1061

bpf_vlog_reset(&env->log, env->log.end_pos - 1);

kernel/bpf/liveness.c

1064

verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]);

kernel/bpf/liveness.c

1065

verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]);

kernel/bpf/liveness.c

1071

verbose(env, "%3d: ", idx);

kernel/bpf/liveness.c

1072

bpf_verbose_insn(env, insn);

kernel/bpf/liveness.c

1073

bpf_vlog_reset(&env->log, env->log.end_pos - 1);

kernel/bpf/liveness.c

1076

verbose(env, "\tfp%+d: ", -(i + 1) * 8); verbose_arg_track(env, &at_stack_in[i]);

kernel/bpf/liveness.c

1077

verbose(env, " -> "); verbose_arg_track(env, &at_stack_out[i]);

kernel/bpf/liveness.c

1080

verbose(env, "\n");

kernel/bpf/liveness.c

109

subprog_start = env->subprog_info[st->frame[frameno]->subprogno].start;

kernel/bpf/liveness.c

1094

static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/liveness.c

11

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/liveness.c

114

f = find_instance(env, key, depth);

kernel/bpf/liveness.c

1184

env->callsite_at_stack[callsites[src->frame]];

kernel/bpf/liveness.c

120

int bpf_stack_liveness_init(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

122

env->liveness = kvzalloc_obj(*env->liveness, GFP_KERNEL_ACCOUNT);

kernel/bpf/liveness.c

123

if (!env->liveness)

kernel/bpf/liveness.c

125

hash_init(env->liveness->func_instances);

kernel/bpf/liveness.c

129

void bpf_stack_liveness_free(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

1334

static int record_load_store_access(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1338

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/liveness.c

135

if (!env->liveness)

kernel/bpf/liveness.c

137

hash_for_each_safe(env->liveness->func_instances, bkt, tmp, instance, hl_node) {

kernel/bpf/liveness.c

1401

static int record_arg_access(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1416

bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx);

kernel/bpf/liveness.c

1418

bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx);

kernel/bpf/liveness.c

142

kvfree(env->liveness);

kernel/bpf/liveness.c

1438

static int record_call_access(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1443

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/liveness.c

1450

if (bpf_get_call_summary(env, insn, &cs))

kernel/bpf/liveness.c

1454

err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx);

kernel/bpf/liveness.c

1460

err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r],

kernel/bpf/liveness.c

1472

static int find_callback_subprog(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1476

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/liveness.c

1525

static void print_subprog_arg_access(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1530

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/liveness.c

1531

int start = env->subprog_info[subprog].start;

kernel/bpf/liveness.c

1535

if (!(env->log.level & BPF_LOG_LEVEL2))

kernel/bpf/liveness.c

1538

verbose(env, "%s:\n", fmt_subprog(env, subprog));

kernel/bpf/liveness.c

1546

verbose(env, "%3d: ", idx);

kernel/bpf/liveness.c

1547

bpf_verbose_insn(env, &insns[idx]);

kernel/bpf/liveness.c

1571

bpf_vlog_reset(&env->log, env->log.end_pos - 1);

kernel/bpf/liveness.c

1572

verbose(env, " //");

kernel/bpf/liveness.c

1579

verbose(env, " r%d=", r);

kernel/bpf/liveness.c

1580

verbose_arg_track(env, &info->at_in[i][r]);

kernel/bpf/liveness.c

1585

verbose(env, " sa%d=", r);

kernel/bpf/liveness.c

1586

verbose_arg_track(env, &info->at_in[i][MAX_BPF_REG + r]);

kernel/bpf/liveness.c

1594

verbose(env, " fp%+d=", -(r + 1) * 8);

kernel/bpf/liveness.c

1595

verbose_arg_track(env, &at_stack_in[i][r]);

kernel/bpf/liveness.c

1599

verbose(env, "\n");

kernel/bpf/liveness.c

1614

static int compute_subprog_args(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1621

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/liveness.c

1623

int start = env->subprog_info[subprog].start;

kernel/bpf/liveness.c

1624

int po_start = env->subprog_info[subprog].postorder_start;

kernel/bpf/liveness.c

1625

int end = env->subprog_info[subprog + 1].start;

kernel/bpf/liveness.c

1626

int po_end = env->subprog_info[subprog + 1].postorder_start;

kernel/bpf/liveness.c

1677

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/liveness.c

1678

verbose(env, "subprog#%d: analyzing (depth %d)...\n", subprog, depth);

kernel/bpf/liveness.c

1684

int idx = env->cfg.insn_postorder[p];

kernel/bpf/liveness.c

1695

arg_track_xfer(env, insn, idx, at_out, at_stack_out,

kernel/bpf/liveness.c

1697

arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out);

kernel/bpf/liveness.c

1700

succ = bpf_insn_successors(env, idx);

kernel/bpf/liveness.c

1711

changed |= arg_track_join(env, idx, target, r,

kernel/bpf/liveness.c

1715

changed |= arg_track_join(env, idx, target, -r - 1,

kernel/bpf/liveness.c

1724

int idx = env->cfg.insn_postorder[p];

kernel/bpf/liveness.c

1728

err = record_load_store_access(env, instance, at_in[i], idx);

kernel/bpf/liveness.c

1733

err = record_call_access(env, instance, at_in[i], idx);

kernel/bpf/liveness.c

1738

if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) {

kernel/bpf/liveness.c

1739

kvfree(env->callsite_at_stack[idx]);

kernel/bpf/liveness.c

1740

env->callsite_at_stack[idx] =

kernel/bpf/liveness.c

1741

kvmalloc_objs(*env->callsite_at_stack[idx],

kernel/bpf/liveness.c

1743

if (!env->callsite_at_stack[idx]) {

kernel/bpf/liveness.c

1747

memcpy(env->callsite_at_stack[idx],

kernel/bpf/liveness.c

1755

print_subprog_arg_access(env, subprog, info, at_stack_in);

kernel/bpf/liveness.c

1847

static int analyze_subprog(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

1855

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/liveness.c

1856

int start = env->subprog_info[subprog].start;

kernel/bpf/liveness.c

1857

int po_start = env->subprog_info[subprog].postorder_start;

kernel/bpf/liveness.c

1858

int po_end = env->subprog_info[subprog + 1].postorder_start;

kernel/bpf/liveness.c

1862

if (++env->liveness->subprog_calls > 10000) {

kernel/bpf/liveness.c

1863

verbose(env, "liveness analysis exceeded complexity limit (%d calls)\n",

kernel/bpf/liveness.c

1864

env->liveness->subprog_calls);

kernel/bpf/liveness.c

1890

err = compute_subprog_args(env, &info[subprog], entry_args, instance, callsites);

kernel/bpf/liveness.c

1896

int idx = env->cfg.insn_postorder[p];

kernel/bpf/liveness.c

1908

callee = bpf_find_subprog(env, target);

kernel/bpf/liveness.c

1917

} else if (bpf_calls_callback(env, idx)) {

kernel/bpf/liveness.c

1918

callee = find_callback_subprog(env, insn, idx, &caller_reg, &cb_callee_reg);

kernel/bpf/liveness.c

1953

callee_instance = call_instance(env, instance, idx, callee);

kernel/bpf/liveness.c

1959

err = analyze_subprog(env, callee_args, info, callee_instance, callsites);

kernel/bpf/liveness.c

1984

update_instance(env, instance);

kernel/bpf/liveness.c

1993

int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

1996

int insn_cnt = env->prog->len;

kernel/bpf/liveness.c

2001

info = kvzalloc_objs(*info, env->subprog_cnt, GFP_KERNEL_ACCOUNT);

kernel/bpf/liveness.c

2005

env->callsite_at_stack = kvzalloc_objs(*env->callsite_at_stack, insn_cnt,

kernel/bpf/liveness.c

2007

if (!env->callsite_at_stack) {

kernel/bpf/liveness.c

2022

for (k = env->subprog_cnt - 1; k >= 0; k--) {

kernel/bpf/liveness.c

2023

int sub = env->subprog_topo_order[k];

kernel/bpf/liveness.c

2025

if (info[sub].at_in && !bpf_subprog_is_global(env, sub))

kernel/bpf/liveness.c

2027

instance = call_instance(env, NULL, 0, sub);

kernel/bpf/liveness.c

2032

err = analyze_subprog(env, NULL, info, instance, callsites);

kernel/bpf/liveness.c

2037

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/liveness.c

2038

err = print_instances(env);

kernel/bpf/liveness.c

2042

kvfree(env->callsite_at_stack[k]);

kernel/bpf/liveness.c

2043

kvfree(env->callsite_at_stack);

kernel/bpf/liveness.c

2044

env->callsite_at_stack = NULL;

kernel/bpf/liveness.c

2045

for (k = 0; k < env->subprog_cnt; k++)

kernel/bpf/liveness.c

2063

static void compute_insn_live_regs(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

218

bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)

kernel/bpf/liveness.c

2182

if (bpf_get_call_summary(env, insn, &cs))

kernel/bpf/liveness.c

2206

int bpf_compute_live_registers(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

2208

struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;

kernel/bpf/liveness.c

2209

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/liveness.c

2211

int insn_cnt = env->prog->len;

kernel/bpf/liveness.c

2241

compute_insn_live_regs(env, &insns[i], &state[i]);

kernel/bpf/liveness.c

2244

err = bpf_compute_subprog_arg_access(env);

kernel/bpf/liveness.c

2251

for (i = 0; i < env->cfg.cur_postorder; ++i) {

kernel/bpf/liveness.c

2252

int insn_idx = env->cfg.insn_postorder[i];

kernel/bpf/liveness.c

2258

succ = bpf_insn_successors(env, insn_idx);

kernel/bpf/liveness.c

2273

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/liveness.c

2274

verbose(env, "Live regs before insn:\n");

kernel/bpf/liveness.c

2276

if (env->insn_aux_data[i].scc)

kernel/bpf/liveness.c

2277

verbose(env, "%3d ", env->insn_aux_data[i].scc);

kernel/bpf/liveness.c

2279

verbose(env, " ");

kernel/bpf/liveness.c

2280

verbose(env, "%3d: ", i);

kernel/bpf/liveness.c

2283

verbose(env, "%d", j);

kernel/bpf/liveness.c

2285

verbose(env, ".");

kernel/bpf/liveness.c

2286

verbose(env, " ");

kernel/bpf/liveness.c

2287

bpf_verbose_insn(env, &insns[i]);

kernel/bpf/liveness.c

245

struct bpf_prog *prog = env->prog;

kernel/bpf/liveness.c

251

jt = env->insn_aux_data[idx].jt;

kernel/bpf/liveness.c

256

succ = env->succ;

kernel/bpf/liveness.c

273

static inline bool update_insn(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

282

succ = bpf_insn_successors(env, insn_idx);

kernel/bpf/liveness.c

307

static void update_instance(struct bpf_verifier_env *env, struct func_instance *instance)

kernel/bpf/liveness.c

310

int *insn_postorder = env->cfg.insn_postorder;

kernel/bpf/liveness.c

315

subprog = &env->subprog_info[instance->subprog];

kernel/bpf/liveness.c

326

changed |= update_insn(env, instance, frame, insn_postorder[i]);

kernel/bpf/liveness.c

339

int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/liveness.c

341

struct live_stack_query *q = &env->liveness->live_stack_query;

kernel/bpf/liveness.c

347

instance = lookup_instance(env, st, frame);

kernel/bpf/liveness.c

360

bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi)

kernel/bpf/liveness.c

368

struct live_stack_query *q = &env->liveness->live_stack_query;

kernel/bpf/liveness.c

398

alive = bpf_calls_callback(env, callsite)

kernel/bpf/liveness.c

408

static char *fmt_subprog(struct bpf_verifier_env *env, int subprog)

kernel/bpf/liveness.c

410

const char *name = env->subprog_info[subprog].name;

kernel/bpf/liveness.c

412

snprintf(env->tmp_str_buf, sizeof(env->tmp_str_buf),

kernel/bpf/liveness.c

414

return env->tmp_str_buf;

kernel/bpf/liveness.c

417

static char *fmt_instance(struct bpf_verifier_env *env, struct func_instance *instance)

kernel/bpf/liveness.c

419

snprintf(env->tmp_str_buf, sizeof(env->tmp_str_buf),

kernel/bpf/liveness.c

421

return env->tmp_str_buf;

kernel/bpf/liveness.c

434

static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis)

kernel/bpf/liveness.c

436

int buf_sz = sizeof(env->tmp_str_buf);

kernel/bpf/liveness.c

437

char *buf = env->tmp_str_buf;

kernel/bpf/liveness.c

475

return env->tmp_str_buf;

kernel/bpf/liveness.c

478

static void print_instance(struct bpf_verifier_env *env, struct func_instance *instance)

kernel/bpf/liveness.c

480

int start = env->subprog_info[instance->subprog].start;

kernel/bpf/liveness.c

481

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/liveness.c

488

if (!(env->log.level & BPF_LOG_LEVEL2))

kernel/bpf/liveness.c

491

verbose(env, "stack use/def %s ", fmt_subprog(env, instance->subprog));

kernel/bpf/liveness.c

492

verbose(env, "%s:\n", fmt_instance(env, instance));

kernel/bpf/liveness.c

497

pos = env->log.end_pos;

kernel/bpf/liveness.c

498

verbose(env, "%3d: ", insn_idx);

kernel/bpf/liveness.c

499

bpf_verbose_insn(env, &insns[insn_idx]);

kernel/bpf/liveness.c

500

bpf_vlog_reset(&env->log, env->log.end_pos - 1); /* remove \n */

kernel/bpf/liveness.c

501

insn_pos = env->log.end_pos;

kernel/bpf/liveness.c

502

verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' ');

kernel/bpf/liveness.c

503

pos = env->log.end_pos;

kernel/bpf/liveness.c

504

verbose(env, " use: ");

kernel/bpf/liveness.c

509

verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read));

kernel/bpf/liveness.c

513

bpf_vlog_reset(&env->log, pos);

kernel/bpf/liveness.c

514

pos = env->log.end_pos;

kernel/bpf/liveness.c

515

verbose(env, " def: ");

kernel/bpf/liveness.c

520

verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write));

kernel/bpf/liveness.c

524

bpf_vlog_reset(&env->log, has_use ? pos : insn_pos);

kernel/bpf/liveness.c

525

verbose(env, "\n");

kernel/bpf/liveness.c

546

static int print_instances(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

549

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

563

print_instance(env, sorted_instances[i]);

kernel/bpf/liveness.c

61

static struct func_instance *find_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

638

static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at)

kernel/bpf/liveness.c

64

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

643

case ARG_NONE: verbose(env, "_"); break;

kernel/bpf/liveness.c

644

case ARG_UNVISITED: verbose(env, "?"); break;

kernel/bpf/liveness.c

645

case ARG_IMPRECISE: verbose(env, "IMP%x", at->mask); break;

kernel/bpf/liveness.c

649

verbose(env, "fp%d ?", at->frame);

kernel/bpf/liveness.c

653

verbose(env, "|");

kernel/bpf/liveness.c

654

verbose(env, "fp%d%+d", at->frame, at->off[i]);

kernel/bpf/liveness.c

74

static struct func_instance *call_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

79

u32 subprog_start = env->subprog_info[subprog].start;

kernel/bpf/liveness.c

795

static bool arg_track_join(struct bpf_verifier_env *env, int idx, int target, int r,

kernel/bpf/liveness.c

805

if (!(env->log.level & BPF_LOG_LEVEL2) || !arg_is_visited(&old))

kernel/bpf/liveness.c

808

verbose(env, "arg JOIN insn %d -> %d ", idx, target);

kernel/bpf/liveness.c

810

verbose(env, "sa%d: ", r - MAX_BPF_REG);

kernel/bpf/liveness.c

812

verbose(env, "r%d: ", r);

kernel/bpf/liveness.c

814

verbose(env, "fp%+d: ", r * 8);

kernel/bpf/liveness.c

815

verbose_arg_track(env, &old);

kernel/bpf/liveness.c

816

verbose(env, " + ");

kernel/bpf/liveness.c

817

verbose_arg_track(env, &out);

kernel/bpf/liveness.c

818

verbose(env, " => ");

kernel/bpf/liveness.c

819

verbose_arg_track(env, &new_val);

kernel/bpf/liveness.c

820

verbose(env, "\n");

kernel/bpf/liveness.c

84

f = find_instance(env, lookup_key, depth);

kernel/bpf/liveness.c

95

f->insn_cnt = (env->subprog_info + subprog + 1)->start - subprog_start;

kernel/bpf/liveness.c

97

hash_add(env->liveness->func_instances, &f->hl_node, hash);

kernel/bpf/log.c

14

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/log.c

304

__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,

kernel/bpf/log.c

309

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/log.c

313

bpf_verifier_vlog(&env->log, fmt, args);

kernel/bpf/log.c

340

__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,

kernel/bpf/log.c

348

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/log.c

351

prev_linfo = env->prev_linfo;

kernel/bpf/log.c

352

linfo = bpf_find_linfo(env->prog, insn_off);

kernel/bpf/log.c

372

bpf_verifier_vlog(&env->log, prefix_fmt, args);

kernel/bpf/log.c

376

btf = env->prog->aux->btf;

kernel/bpf/log.c

378

verbose(env, "%s", s); /* source code line */

kernel/bpf/log.c

384

verbose(env, " @ %s:%u\n", fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col));

kernel/bpf/log.c

386

env->prev_linfo = linfo;

kernel/bpf/log.c

399

const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)

kernel/bpf/log.c

445

snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",

kernel/bpf/log.c

447

return env->tmp_str_buf;

kernel/bpf/log.c

522

static void verbose_unum(struct bpf_verifier_env *env, u64 num)

kernel/bpf/log.c

525

verbose(env, "%llu", num);

kernel/bpf/log.c

527

verbose(env, "%#llx", num);

kernel/bpf/log.c

530

static void verbose_snum(struct bpf_verifier_env *env, s64 num)

kernel/bpf/log.c

533

verbose(env, "%lld", num);

kernel/bpf/log.c

535

verbose(env, "%#llx", num);

kernel/bpf/log.c

553

static void print_scalar_ranges(struct bpf_verifier_env *env,

kernel/bpf/log.c

597

verbose(env, "%s%s=", *sep, m1->name);

kernel/bpf/log.c

608

verbose(env, "%s=", m2->name);

kernel/bpf/log.c

612

verbose_snum(env, m1->val);

kernel/bpf/log.c

614

verbose_unum(env, m1->val);

kernel/bpf/log.c

633

#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })

kernel/bpf/log.c

635

static void print_reg_state(struct bpf_verifier_env *env,

kernel/bpf/log.c

644

verbose(env, "P");

kernel/bpf/log.c

646

verbose_snum(env, reg->var_off.value);

kernel/bpf/log.c

650

verbose(env, "%s", reg_type_str(env, t));

kernel/bpf/log.c

655

verbose(env, "[%d]", reg->frameno);

kernel/bpf/log.c

657

verbose_snum(env, reg->var_off.value + reg->delta);

kernel/bpf/log.c

662

verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));

kernel/bpf/log.c

663

verbose(env, "(");

kernel/bpf/log.c

667

verbose(env, "%+d", reg->delta);

kernel/bpf/log.c

681

verbose_snum(env, reg->delta);

kernel/bpf/log.c

685

verbose_unum(env, reg->range);

kernel/bpf/log.c

689

verbose_unum(env, reg->mem_size);

kernel/bpf/log.c

697

verbose_snum(env, reg->var_off.value);

kernel/bpf/log.c

700

print_scalar_ranges(env, reg, &sep);

kernel/bpf/log.c

708

verbose(env, ")");

kernel/bpf/log.c

711

void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

kernel/bpf/log.c

719

verbose(env, " frame%d:", state->frameno);

kernel/bpf/log.c

724

if (!print_all && !reg_scratched(env, i))

kernel/bpf/log.c

726

verbose(env, " R%d", i);

kernel/bpf/log.c

727

verbose(env, "=");

kernel/bpf/log.c

728

print_reg_state(env, state, reg);

kernel/bpf/log.c

737

if (!print_all && !stack_slot_scratched(env, i))

kernel/bpf/log.c

759

verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);

kernel/bpf/log.c

760

print_reg_state(env, state, reg);

kernel/bpf/log.c

767

verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);

kernel/bpf/log.c

768

verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type));

kernel/bpf/log.c

773

verbose(env, ")");

kernel/bpf/log.c

780

verbose(env, " fp%d=iter_%s(id=%d,state=%s,depth=%u)",

kernel/bpf/log.c

789

verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);

kernel/bpf/log.c

794

verbose(env, " refs=%d", vstate->refs[0].id);

kernel/bpf/log.c

797

verbose(env, ",%d", vstate->refs[i].id);

kernel/bpf/log.c

800

verbose(env, " cb");

kernel/bpf/log.c

802

verbose(env, " async_cb");

kernel/bpf/log.c

803

verbose(env, "\n");

kernel/bpf/log.c

805

mark_verifier_state_clean(env);

kernel/bpf/log.c

814

void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

kernel/bpf/log.c

817

if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {

kernel/bpf/log.c

819

bpf_vlog_reset(&env->log, env->prev_log_pos - 1);

kernel/bpf/log.c

820

verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' ');

kernel/bpf/log.c

822

verbose(env, "%d:", env->insn_idx);

kernel/bpf/log.c

824

print_verifier_state(env, vstate, frameno, false);

kernel/bpf/offload.c

303

int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,

kernel/bpf/offload.c

310

offload = env->prog->aux->offload;

kernel/bpf/offload.c

312

ret = offload->offdev->ops->insn_hook(env, insn_idx,

kernel/bpf/offload.c

319

int bpf_prog_offload_finalize(struct bpf_verifier_env *env)

kernel/bpf/offload.c

325

offload = env->prog->aux->offload;

kernel/bpf/offload.c

328

ret = offload->offdev->ops->finalize(env);

kernel/bpf/offload.c

338

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

kernel/bpf/offload.c

346

offload = env->prog->aux->offload;

kernel/bpf/offload.c

350

ret = ops->replace_insn(env, off, insn);

kernel/bpf/offload.c

357

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

kernel/bpf/offload.c

363

offload = env->prog->aux->offload;

kernel/bpf/offload.c

366

ret = offload->offdev->ops->remove_insns(env, off, cnt);

kernel/bpf/states.c

100

struct bpf_scc_info *info = env->scc_info[callchain->scc];

kernel/bpf/states.c

1001

if (!refsafe(old, cur, &env->idmap_scratch))

kernel/bpf/states.c

1011

if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))

kernel/bpf/states.c

1020

static int propagate_precision(struct bpf_verifier_env *env,

kernel/bpf/states.c

1038

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/states.c

1040

verbose(env, "frame %d: propagating r%d", fr, i);

kernel/bpf/states.c

1042

verbose(env, ",r%d", i);

kernel/bpf/states.c

1044

bpf_bt_set_frame_reg(&env->bt, fr, i);

kernel/bpf/states.c

1055

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/states.c

1057

verbose(env, "frame %d: propagating fp%d",

kernel/bpf/states.c

1060

verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);

kernel/bpf/states.c

1062

bpf_bt_set_frame_slot(&env->bt, fr, i);

kernel/bpf/states.c

1065

if (!first && (env->log.level & BPF_LOG_LEVEL2))

kernel/bpf/states.c

1066

verbose(env, "\n");

kernel/bpf/states.c

1069

err = bpf_mark_chain_precision(env, cur, -1, changed);

kernel/bpf/states.c

1084

static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit)

kernel/bpf/states.c

1094

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/states.c

1095

verbose(env, "%s: too many iterations\n", __func__);

kernel/bpf/states.c

1097

bpf_mark_all_scalars_precise(env, &backedge->state);

kernel/bpf/states.c

1103

err = propagate_precision(env, st->equal_state, st, &changed);

kernel/bpf/states.c

116

static struct bpf_scc_visit *scc_visit_alloc(struct bpf_verifier_env *env,

kernel/bpf/states.c

12

static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/states.c

1213

static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/states.c

1238

int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/states.c

1242

struct bpf_verifier_state *cur = env->cur_state, *new;

kernel/bpf/states.c

1247

force_new_state = env->test_state_freq || bpf_is_force_checkpoint(env, insn_idx) ||

kernel/bpf/states.c

125

info = env->scc_info[scc];

kernel/bpf/states.c

1260

if (env->jmps_processed - env->prev_jmps_processed >= 2 &&

kernel/bpf/states.c

1261

env->insn_processed - env->prev_insn_processed >= 8)

kernel/bpf/states.c

1265

err = clean_verifier_state(env, cur);

kernel/bpf/states.c

1270

head = bpf_explored_state(env, insn_idx);

kernel/bpf/states.c

128

info = kvrealloc(env->scc_info[scc], new_sz, GFP_KERNEL_ACCOUNT);

kernel/bpf/states.c

131

env->scc_info[scc] = info;

kernel/bpf/states.c

1331

if (is_iter_next_insn(env, insn_idx)) {

kernel/bpf/states.c

1332

if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/states.c

1347

iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr;

kernel/bpf/states.c

1355

if (is_may_goto_insn_at(env, insn_idx)) {

kernel/bpf/states.c

1357

states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/states.c

1362

if (bpf_calls_callback(env, insn_idx)) {

kernel/bpf/states.c

1363

if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/states.c

1371

states_equal(env, &sl->state, cur, EXACT) &&

kernel/bpf/states.c

1375

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/states.c

1376

verbose(env, "infinite loop detected at insn %d\n", insn_idx);

kernel/bpf/states.c

1377

verbose(env, "cur state:");

kernel/bpf/states.c

1378

print_verifier_state(env, cur, cur->curframe, true);

kernel/bpf/states.c

1379

verbose(env, "old state:");

kernel/bpf/states.c

1380

print_verifier_state(env, &sl->state, cur->curframe, true);

kernel/bpf/states.c

1397

env->jmps_processed - env->prev_jmps_processed < 20 &&

kernel/bpf/states.c

1398

env->insn_processed - env->prev_insn_processed < 100)

kernel/bpf/states.c

14

return bpf_is_may_goto_insn(&env->prog->insnsi[insn_idx]);

kernel/bpf/states.c

140

static char *format_callchain(struct bpf_verifier_env *env, struct bpf_scc_callchain *callchain)

kernel/bpf/states.c

1403

loop = incomplete_read_marks(env, &sl->state);

kernel/bpf/states.c

1404

if (states_equal(env, &sl->state, cur, loop ? RANGE_WITHIN : NOT_EXACT)) {

kernel/bpf/states.c

1414

if (bpf_is_jmp_point(env, env->insn_idx))

kernel/bpf/states.c

1415

err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0);

kernel/bpf/states.c

1416

err = err ? : propagate_precision(env, &sl->state, cur, NULL);

kernel/bpf/states.c

142

char *buf = env->tmp_str_buf;

kernel/bpf/states.c

1501

err = err ?: add_scc_backedge(env, &sl->state, backedge);

kernel/bpf/states.c

1527

n = bpf_is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;

kernel/bpf/states.c

153

return env->tmp_str_buf;

kernel/bpf/states.c

1534

list_add(&sl->node, &env->free_list);

kernel/bpf/states.c

1535

env->free_list_size++;

kernel/bpf/states.c

1536

env->explored_states_size--;

kernel/bpf/states.c

1537

maybe_free_verifier_state(env, sl);

kernel/bpf/states.c

1541

if (env->max_states_per_insn < states_cnt)

kernel/bpf/states.c

1542

env->max_states_per_insn = states_cnt;

kernel/bpf/states.c

1544

if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)

kernel/bpf/states.c

1562

env->total_states++;

kernel/bpf/states.c

1563

env->explored_states_size++;

kernel/bpf/states.c

1564

update_peak_states(env);

kernel/bpf/states.c

1565

env->prev_jmps_processed = env->jmps_processed;

kernel/bpf/states.c

1566

env->prev_insn_processed = env->insn_processed;

kernel/bpf/states.c

1569

if (env->bpf_capable)

kernel/bpf/states.c

1570

mark_all_scalars_imprecise(env, cur);

kernel/bpf/states.c

1572

bpf_clear_singular_ids(env, cur);

kernel/bpf/states.c

1583

verifier_bug_if(new->branches != 1, env,

kernel/bpf/states.c

1586

err = maybe_enter_scc(env, new);

kernel/bpf/states.c

160

static int maybe_enter_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/states.c

162

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/states.c

165

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/states.c

167

visit = scc_visit_lookup(env, callchain);

kernel/bpf/states.c

168

visit = visit ?: scc_visit_alloc(env, callchain);

kernel/bpf/states.c

17

static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/states.c

173

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/states.c

174

verbose(env, "SCC enter %s\n", format_callchain(env, callchain));

kernel/bpf/states.c

179

static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit);

kernel/bpf/states.c

185

static int maybe_exit_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/states.c

187

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/states.c

19

return env->insn_aux_data[insn_idx].is_iter_next;

kernel/bpf/states.c

190

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/states.c

192

visit = scc_visit_lookup(env, callchain);

kernel/bpf/states.c

207

verifier_bug(env, "scc exit: no visit info for call chain %s",

kernel/bpf/states.c

208

format_callchain(env, callchain));

kernel/bpf/states.c

215

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/states.c

216

verbose(env, "SCC exit %s\n", format_callchain(env, callchain));

kernel/bpf/states.c

218

env->num_backedges -= visit->num_backedges;

kernel/bpf/states.c

22

static void update_peak_states(struct bpf_verifier_env *env)

kernel/bpf/states.c

220

update_peak_states(env);

kernel/bpf/states.c

221

return propagate_backedges(env, visit);

kernel/bpf/states.c

227

static int add_scc_backedge(struct bpf_verifier_env *env,

kernel/bpf/states.c

231

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/states.c

234

if (!compute_scc_callchain(env, st, callchain)) {

kernel/bpf/states.c

235

verifier_bug(env, "add backedge: no SCC in verification path, insn_idx %d",

kernel/bpf/states.c

239

visit = scc_visit_lookup(env, callchain);

kernel/bpf/states.c

241

verifier_bug(env, "add backedge: no visit info for call chain %s",

kernel/bpf/states.c

242

format_callchain(env, callchain));

kernel/bpf/states.c

245

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/states.c

246

verbose(env, "SCC backedge %s\n", format_callchain(env, callchain));

kernel/bpf/states.c

250

env->num_backedges++;

kernel/bpf/states.c

251

update_peak_states(env);

kernel/bpf/states.c

259

static bool incomplete_read_marks(struct bpf_verifier_env *env,

kernel/bpf/states.c

26

cur_states = env->explored_states_size + env->free_list_size + env->num_backedges;

kernel/bpf/states.c

262

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/states.c

265

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/states.c

267

visit = scc_visit_lookup(env, callchain);

kernel/bpf/states.c

27

env->peak_states = max(env->peak_states, cur_states);

kernel/bpf/states.c

273

int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/states.c

285

verifier_bug_if((int)br < 0, env, "%s:branches_to_explore=%d", __func__, br);

kernel/bpf/states.c

288

err = maybe_exit_scc(env, st);

kernel/bpf/states.c

294

maybe_free_verifier_state(env, sl);

kernel/bpf/states.c

398

static void __clean_func_state(struct bpf_verifier_env *env,

kernel/bpf/states.c

41

static bool incomplete_read_marks(struct bpf_verifier_env *env,

kernel/bpf/states.c

410

bpf_mark_reg_not_init(env, &st->regs[i]);

kernel/bpf/states.c

419

bool lo_live = bpf_stack_slot_alive(env, frame, i * 2);

kernel/bpf/states.c

420

bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1);

kernel/bpf/states.c

466

bpf_mark_reg_not_init(env, spill);

kernel/bpf/states.c

474

static int clean_verifier_state(struct bpf_verifier_env *env,

kernel/bpf/states.c

479

err = bpf_live_stack_query_init(env, st);

kernel/bpf/states.c

48

static void maybe_free_verifier_state(struct bpf_verifier_env *env,

kernel/bpf/states.c

484

u16 live_regs = env->insn_aux_data[ip].live_regs_before;

kernel/bpf/states.c

486

__clean_func_state(env, st->frame[i], live_regs, i);

kernel/bpf/states.c

507

static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,

kernel/bpf/states.c

53

|| incomplete_read_marks(env, &sl->state))

kernel/bpf/states.c

544

if (env->explore_alu_limits) {

kernel/bpf/states.c

58

env->free_list_size--;

kernel/bpf/states.c

674

static bool is_stack_misc_after(struct bpf_verifier_env *env,

kernel/bpf/states.c

682

env->allow_uninit_stack))

kernel/bpf/states.c

690

static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,

kernel/bpf/states.c

696

if (is_stack_misc_after(env, stack, im))

kernel/bpf/states.c

702

static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,

kernel/bpf/states.c

736

if (env->allow_uninit_stack &&

kernel/bpf/states.c

74

static bool compute_scc_callchain(struct bpf_verifier_env *env,

kernel/bpf/states.c

753

old_reg = scalar_reg_for_stack(env, &old->stack[spi], im);

kernel/bpf/states.c

754

cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im);

kernel/bpf/states.c

756

if (!regsafe(env, old_reg, cur_reg, idmap, exact))

kernel/bpf/states.c

793

if (!regsafe(env, &old->stack[spi].spilled_ptr,

kernel/bpf/states.c

8

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/states.c

83

scc = env->insn_aux_data[insn_idx].scc;

kernel/bpf/states.c

846

static bool stack_arg_safe(struct bpf_verifier_env *env, struct bpf_func_state *old,

kernel/bpf/states.c

861

if (!regsafe(env, old_arg, cur_arg, idmap, exact))

kernel/bpf/states.c

944

static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,

kernel/bpf/states.c

947

u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;

kernel/bpf/states.c

958

!regsafe(env, &old->regs[i], &cur->regs[i],

kernel/bpf/states.c

959

&env->idmap_scratch, exact))

kernel/bpf/states.c

962

if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))

kernel/bpf/states.c

965

if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact))

kernel/bpf/states.c

97

static struct bpf_scc_visit *scc_visit_lookup(struct bpf_verifier_env *env,

kernel/bpf/states.c

971

static void reset_idmap_scratch(struct bpf_verifier_env *env)

kernel/bpf/states.c

973

struct bpf_idmap *idmap = &env->idmap_scratch;

kernel/bpf/states.c

975

idmap->tmp_id_gen = env->id_gen;

kernel/bpf/states.c

979

static bool states_equal(struct bpf_verifier_env *env,

kernel/bpf/states.c

990

reset_idmap_scratch(env);

kernel/bpf/verifier.c

10006

static int check_bpf_snprintf_call(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10029

verbose(env, "failed to retrieve map value address\n");

kernel/bpf/verifier.c

1003

static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10039

verbose(env, "Invalid format string\n");

kernel/bpf/verifier.c

10044

static int check_get_func_ip(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10046

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

10050

if (!bpf_prog_has_trampoline(env->prog)) {

kernel/bpf/verifier.c

10051

verbose(env, "func %s#%d supported only for fentry/fexit/fsession/fmod_ret programs\n",

kernel/bpf/verifier.c

1006

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

10060

verbose(env, "func %s#%d not supported for program type %d\n",

kernel/bpf/verifier.c

10065

static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10067

return &env->insn_aux_data[env->insn_idx];

kernel/bpf/verifier.c

10070

static bool loop_flag_is_zero(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10072

struct bpf_reg_state *reg = reg_state(env, BPF_REG_4);

kernel/bpf/verifier.c

10076

mark_chain_precision(env, BPF_REG_4);

kernel/bpf/verifier.c

10081

static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)

kernel/bpf/verifier.c

10083

struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;

kernel/bpf/verifier.c

10087

state->fit_for_inline = loop_flag_is_zero(env);

kernel/bpf/verifier.c

10095

state->fit_for_inline = (loop_flag_is_zero(env) &&

kernel/bpf/verifier.c

10117

int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,

kernel/bpf/verifier.c

10123

if (!env->ops->get_func_proto)

kernel/bpf/verifier.c

10126

*ptr = env->ops->get_func_proto(func_id, env->prog);

kernel/bpf/verifier.c

1013

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

10131

static inline bool in_sleepable_context(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10133

return !env->cur_state->active_rcu_locks &&

kernel/bpf/verifier.c

10134

!env->cur_state->active_preempt_locks &&

kernel/bpf/verifier.c

10135

!env->cur_state->active_locks &&

kernel/bpf/verifier.c

10136

!env->cur_state->active_irq_id &&

kernel/bpf/verifier.c

10137

in_sleepable(env);

kernel/bpf/verifier.c

10140

static const char *non_sleepable_context_description(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10142

if (env->cur_state->active_rcu_locks)

kernel/bpf/verifier.c

10144

if (env->cur_state->active_preempt_locks)

kernel/bpf/verifier.c

10146

if (env->cur_state->active_irq_id)

kernel/bpf/verifier.c

10148

if (env->cur_state->active_locks)

kernel/bpf/verifier.c

10153

static int release_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

10162

err = unmark_stack_slots_dynptr(env, reg);

kernel/bpf/verifier.c

10164

err = ref_convert_alloc_rcu_protected(env, reg->id);

kernel/bpf/verifier.c

10165

else if (reg_is_referenced(env, reg))

kernel/bpf/verifier.c

10166

err = release_reference(env, reg->id);

kernel/bpf/verifier.c

10171

static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

10174

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

10187

err = bpf_get_helper_proto(env, insn->imm, &fn);

kernel/bpf/verifier.c

10189

verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

10194

verbose(env, "program of this type cannot use helper %s#%d\n",

kernel/bpf/verifier.c

10200

if (!env->prog->gpl_compatible && fn->gpl_only) {

kernel/bpf/verifier.c

10201

verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");

kernel/bpf/verifier.c

10205

if (fn->allowed && !fn->allowed(env->prog)) {

kernel/bpf/verifier.c

10206

verbose(env, "helper call is not allowed in probe\n");

kernel/bpf/verifier.c

10213

verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

10222

verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

10226

if (fn->might_sleep && !in_sleepable_context(env)) {

kernel/bpf/verifier.c

10227

verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id,

kernel/bpf/verifier.c

10228

non_sleepable_context_description(env));

kernel/bpf/verifier.c

10233

if (!in_sleepable_context(env))

kernel/bpf/verifier.c

10234

env->insn_aux_data[insn_idx].non_sleepable = true;

kernel/bpf/verifier.c

10239

err = check_func_arg(env, i, &meta, fn, insn_idx);

kernel/bpf/verifier.c

10244

err = record_func_map(env, &meta, func_id, insn_idx);

kernel/bpf/verifier.c

10248

err = record_func_key(env, &meta, func_id, insn_idx);

kernel/bpf/verifier.c

10252

regs = cur_regs(env);

kernel/bpf/verifier.c

10258

err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B,

kernel/bpf/verifier.c

10266

bool convert_rcu = (func_id == BPF_FUNC_kptr_xchg) && in_rcu_cs(env) &&

kernel/bpf/verifier.c

10269

err = release_reg(env, reg, convert_rcu, !!meta.dynptr.id);

kernel/bpf/verifier.c

10276

err = check_resource_leak(env, false, true, "tail_call");

kernel/bpf/verifier.c

10285

verbose(env, "get_local_storage() doesn't support non-zero flags\n");

kernel/bpf/verifier.c

10290

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

10294

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

10298

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

1030

static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

10302

err = check_bpf_snprintf_call(env, regs);

kernel/bpf/verifier.c

10305

update_loop_inline_state(env, meta.subprogno);

kernel/bpf/verifier.c

10309

err = mark_chain_precision(env, BPF_REG_1);

kernel/bpf/verifier.c

10312

if (cur_func(env)->callback_depth < reg_umax(&regs[BPF_REG_1])) {

kernel/bpf/verifier.c

10313

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

10316

cur_func(env)->callback_depth = 0;

kernel/bpf/verifier.c

10317

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

10318

verbose(env, "frame%d bpf_loop iteration limit reached\n",

kernel/bpf/verifier.c

10319

env->cur_state->curframe);

kernel/bpf/verifier.c

10324

verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",

kernel/bpf/verifier.c

10325

reg_type_str(env, regs[BPF_REG_1].type));

kernel/bpf/verifier.c

1033

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

10339

verbose(env, "R1 is not a scalar\n");

kernel/bpf/verifier.c

10345

env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT)

kernel/bpf/verifier.c

10349

env->prog->expected_attach_type == BPF_LSM_CGROUP) {

kernel/bpf/verifier.c

10350

if (!env->prog->aux->attach_func_proto->type) {

kernel/bpf/verifier.c

10354

verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");

kernel/bpf/verifier.c

10357

bpf_lsm_get_retval_range(env->prog, &range);

kernel/bpf/verifier.c

1036

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

10360

err = mark_chain_precision(env, BPF_REG_1);

kernel/bpf/verifier.c

10365

verbose_invalid_scalar(env, r1, range, "At bpf_set_retval", "R1");

kernel/bpf/verifier.c

10396

verbose(env, "Helper has invalid btf/btf_id in R1\n");

kernel/bpf/verifier.c

10400

env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;

kernel/bpf/verifier.c

10405

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

10415

bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);

kernel/bpf/verifier.c

10416

check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

10418

invalidate_outgoing_stack_args(env, cur_func(env));

kernel/bpf/verifier.c

10430

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10437

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10443

verifier_bug(env, "unexpected null map_ptr");

kernel/bpf/verifier.c

10458

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

10462

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10466

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10470

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10474

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10482

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10493

verbose(env, "unable to resolve the size of type '%s': %ld\n",

kernel/bpf/verifier.c

10522

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10534

verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type",

kernel/bpf/verifier.c

10542

verbose(env, "invalid return type %u of func %s#%d\n",

kernel/bpf/verifier.c

10552

verbose(env, "unknown return type %u of func %s#%d\n",

kernel/bpf/verifier.c

10558

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

10561

find_reference_state(env->cur_state, meta.ref_obj.id)) {

kernel/bpf/verifier.c

10565

err = validate_ref_obj(env, &meta.ref_obj);

kernel/bpf/verifier.c

10577

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

10588

int id = acquire_reference(env, insn_idx, 0);

kernel/bpf/verifier.c

10599

err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);

kernel/bpf/verifier.c

10603

err = check_map_func_compatibility(env, meta.map.ptr, func_id);

kernel/bpf/verifier.c

10609

!env->prog->has_callchain_buf) {

kernel/bpf/verifier.c

1062

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);

kernel/bpf/verifier.c

10620

verbose(env, err_str, func_id_name(func_id), func_id);

kernel/bpf/verifier.c

10624

env->prog->has_callchain_buf = true;

kernel/bpf/verifier.c

10628

env->prog->call_get_stack = true;

kernel/bpf/verifier.c

10631

if (check_get_func_ip(env))

kernel/bpf/verifier.c

10633

env->prog->call_get_func_ip = true;

kernel/bpf/verifier.c

10637

if (env->cur_state->curframe) {

kernel/bpf/verifier.c

10640

mark_reg_scratched(env, BPF_REG_0);

kernel/bpf/verifier.c

10641

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

10644

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

10645

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

10646

err = prepare_func_exit(env, &env->insn_idx);

kernel/bpf/verifier.c

10649

env->insn_idx--;

kernel/bpf/verifier.c

1065

static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10656

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

10663

static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs,

kernel/bpf/verifier.c

10671

DEF_NOT_SUBREG : env->insn_idx + 1;

kernel/bpf/verifier.c

10674

mark_insn_zext(env, reg);

kernel/bpf/verifier.c

10678

static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

10681

return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size);

kernel/bpf/verifier.c

1070

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

1075

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

1079

id = acquire_irq_state(env, insn_idx);

kernel/bpf/verifier.c

10901

static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,

kernel/bpf/verifier.c

1094

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

10941

static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10958

verbose(env, "max struct nesting depth exceeded\n");

kernel/bpf/verifier.c

10961

if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))

kernel/bpf/verifier.c

1098

static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

1101

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

1106

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

1117

verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n",

kernel/bpf/verifier.c

1122

err = release_irq_state(env->cur_state, st->id);

kernel/bpf/verifier.c

11258

get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller,

kernel/bpf/verifier.c

1127

for (int i = 0; i < env->cur_state->acquired_refs; i++) {

kernel/bpf/verifier.c

1128

if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) {

kernel/bpf/verifier.c

11281

if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))

kernel/bpf/verifier.c

1129

insn_idx = env->cur_state->refs[i].insn_idx;

kernel/bpf/verifier.c

11335

verbose(env, "kernel function %s %s pointer type %s %s is not supported\n",

kernel/bpf/verifier.c

11336

meta->func_name, reg_arg_name(env, argno),

kernel/bpf/verifier.c

1134

verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n",

kernel/bpf/verifier.c

11343

if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))

kernel/bpf/verifier.c

1135

env->cur_state->active_irq_id, insn_idx);

kernel/bpf/verifier.c

11351

if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&

kernel/bpf/verifier.c

11353

verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n",

kernel/bpf/verifier.c

11354

reg_arg_name(env, argno),

kernel/bpf/verifier.c

11361

static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1139

bpf_mark_reg_not_init(env, st);

kernel/bpf/verifier.c

11408

if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) ||

kernel/bpf/verifier.c

11409

btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))

kernel/bpf/verifier.c

11416

struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value,

kernel/bpf/verifier.c

11424

verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n",

kernel/bpf/verifier.c

11425

meta->func_name, reg_arg_name(env, argno),

kernel/bpf/verifier.c

11426

btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno),

kernel/bpf/verifier.c

11433

static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

1144

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

11450

verifier_bug(env, "unknown irq flags kfunc");

kernel/bpf/verifier.c

11455

if (!is_irq_flag_reg_valid_uninit(env, reg)) {

kernel/bpf/verifier.c

11456

verbose(env, "expected uninitialized irq flag as %s\n",

kernel/bpf/verifier.c

11457

reg_arg_name(env, argno));

kernel/bpf/verifier.c

11461

err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW,

kernel/bpf/verifier.c

11466

err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class);

kernel/bpf/verifier.c

11470

err = is_irq_flag_reg_valid_init(env, reg);

kernel/bpf/verifier.c

11472

verbose(env, "expected an initialized irq flag as %s\n",

kernel/bpf/verifier.c

11473

reg_arg_name(env, argno));

kernel/bpf/verifier.c

11477

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

1148

static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

11481

mark_stack_slots_scratched(env, spi, 1);

kernel/bpf/verifier.c

11483

err = unmark_stack_slot_irq_flag(env, reg, kfunc_class);

kernel/bpf/verifier.c

11491

static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

11495

if (!env->cur_state->active_locks) {

kernel/bpf/verifier.c

11496

verifier_bug(env, "%s w/o active lock", __func__);

kernel/bpf/verifier.c

1150

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

11501

verifier_bug(env, "NON_OWN_REF already set");

kernel/bpf/verifier.c

11512

static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id)

kernel/bpf/verifier.c

11517

WARN_ON_ONCE(release_reference_nomark(env->cur_state, id));

kernel/bpf/verifier.c

11519

bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({

kernel/bpf/verifier.c

11522

ref_set_non_owning(env, reg);

kernel/bpf/verifier.c

11573

static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

1158

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

11587

verifier_bug(env, "unknown reg type for lock check");

kernel/bpf/verifier.c

11592

if (!env->cur_state->active_locks)

kernel/bpf/verifier.c

11594

s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr);

kernel/bpf/verifier.c

11596

verbose(env, "held lock and object are not in the same allocation\n");

kernel/bpf/verifier.c

11700

static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11714

verbose(env, "verifier internal error: unexpected graph root argument type %s\n",

kernel/bpf/verifier.c

1172

static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

11720

verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",

kernel/bpf/verifier.c

11725

static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1174

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

11745

verbose(env, "verifier internal error: unexpected graph node argument type %s\n",

kernel/bpf/verifier.c

11751

verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",

kernel/bpf/verifier.c

11757

__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11769

verifier_bug(env, "unexpected btf mismatch in kfunc call");

kernel/bpf/verifier.c

11773

if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))

kernel/bpf/verifier.c

11778

verbose(env,

kernel/bpf/verifier.c

11780

reg_arg_name(env, argno), head_type_name);

kernel/bpf/verifier.c

11788

verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);

kernel/bpf/verifier.c

1179

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

11793

if (check_reg_allocation_locked(env, reg)) {

kernel/bpf/verifier.c

11794

verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",

kernel/bpf/verifier.c

11800

verifier_bug(env, "repeating %s arg", head_type_name);

kernel/bpf/verifier.c

11807

static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11811

return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD,

kernel/bpf/verifier.c

11815

static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11819

return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT,

kernel/bpf/verifier.c

11824

__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11837

verifier_bug(env, "unexpected btf mismatch in kfunc call");

kernel/bpf/verifier.c

11841

if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))

kernel/bpf/verifier.c

11846

verbose(env,

kernel/bpf/verifier.c

11848

reg_arg_name(env, argno), node_type_name);

kernel/bpf/verifier.c

11855

verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);

kernel/bpf/verifier.c

11863

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,

kernel/bpf/verifier.c

11865

verbose(env, "operation on %s expects arg#1 %s at offset=%d "

kernel/bpf/verifier.c

11878

verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",

kernel/bpf/verifier.c

11888

static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11892

return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,

kernel/bpf/verifier.c

11897

static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11901

return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,

kernel/bpf/verifier.c

11912

static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11914

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

11920

if (env->prog->expected_attach_type == BPF_TRACE_ITER)

kernel/bpf/verifier.c

11924

return in_sleepable(env);

kernel/bpf/verifier.c

11928

static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,

kernel/bpf/verifier.c

11932

struct bpf_func_state *caller = cur_func(env);

kernel/bpf/verifier.c

11933

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

11943

verbose(env, "Function %s has %d > %d args\n", func_name, nargs,

kernel/bpf/verifier.c

11948

verbose(env, "JIT does not support kfunc %s() with %d args\n",

kernel/bpf/verifier.c

11953

ret = check_outgoing_stack_args(env, caller, nargs);

kernel/bpf/verifier.c

11974

verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc");

kernel/bpf/verifier.c

11978

verbose(env, "%s prog->aux cannot be a stack argument\n",

kernel/bpf/verifier.c

11979

reg_arg_name(env, argno));

kernel/bpf/verifier.c

11983

cur_aux(env)->arg_prog = regno;

kernel/bpf/verifier.c

11994

verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

12000

verifier_bug(env, "only one constant argument permitted");

kernel/bpf/verifier.c

12004

verbose(env, "%s must be a known constant\n",

kernel/bpf/verifier.c

12005

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12009

ret = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

12011

ret = mark_stack_arg_precision(env, i);

kernel/bpf/verifier.c

12025

verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");

kernel/bpf/verifier.c

12030

verbose(env, "%s is not a const\n",

kernel/bpf/verifier.c

12031

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12037

ret = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

12039

ret = mark_stack_arg_precision(env, i);

kernel/bpf/verifier.c

12047

verbose(env, "Unrecognized %s type %s\n",

kernel/bpf/verifier.c

12048

reg_arg_name(env, argno), btf_type_str(t));

kernel/bpf/verifier.c

12054

verbose(env, "Possibly NULL pointer passed to trusted %s\n",

kernel/bpf/verifier.c

12055

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12060

!reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) {

kernel/bpf/verifier.c

12061

verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n",

kernel/bpf/verifier.c

12062

func_name, reg_arg_name(env, argno));

kernel/bpf/verifier.c

12066

if (reg_is_referenced(env, reg))

kernel/bpf/verifier.c

12072

kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname,

kernel/bpf/verifier.c

12082

verbose(env, "pointer in %s isn't map pointer\n",

kernel/bpf/verifier.c

12083

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12103

verbose(env,

kernel/bpf/verifier.c

12108

verbose(env,

kernel/bpf/verifier.c

12119

if (!is_trusted_reg(env, reg)) {

kernel/bpf/verifier.c

12121

verbose(env, "%s must be referenced or trusted\n",

kernel/bpf/verifier.c

12122

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12126

verbose(env, "%s must be a rcu pointer\n",

kernel/bpf/verifier.c

12127

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12158

verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);

kernel/bpf/verifier.c

12164

ret = __check_func_arg_reg_off(env, reg, argno, arg_type,

kernel/bpf/verifier.c

12172

verbose(env, "%s expected pointer to ctx, but got %s\n",

kernel/bpf/verifier.c

12173

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

12178

ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));

kernel/bpf/verifier.c

12187

verbose(env, "%s expected for bpf_obj_drop()\n",

kernel/bpf/verifier.c

12188

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12193

verbose(env, "%s expected for bpf_percpu_obj_drop()\n",

kernel/bpf/verifier.c

12194

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12198

verbose(env, "%s expected pointer to allocated object\n",

kernel/bpf/verifier.c

12199

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12202

if (!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12203

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

12233

verifier_bug(env, "no dynptr type for parent of clone");

kernel/bpf/verifier.c

12240

ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type,

kernel/bpf/verifier.c

12248

if (!check_css_task_iter_allowlist(env)) {

kernel/bpf/verifier.c

12249

verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");

kernel/bpf/verifier.c

12253

ret = process_iter_arg(env, reg, argno, insn_idx, meta);

kernel/bpf/verifier.c

12260

verbose(env, "%s expected pointer to map value or allocated object\n",

kernel/bpf/verifier.c

12261

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12265

!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12266

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

12269

ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta);

kernel/bpf/verifier.c

12276

verbose(env, "%s expected pointer to map value or allocated object\n",

kernel/bpf/verifier.c

12277

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12281

!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12282

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

12285

ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta);

kernel/bpf/verifier.c

12291

type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12298

verbose(env, "%s expected pointer to allocated object\n",

kernel/bpf/verifier.c

12299

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12302

if (!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12303

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

12307

ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta);

kernel/bpf/verifier.c

12314

verbose(env, "%s expected pointer to allocated object\n",

kernel/bpf/verifier.c

12315

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12318

if (!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12319

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

12324

!reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

12325

verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);

kernel/bpf/verifier.c

12328

if (in_rbtree_lock_required_cb(env)) {

kernel/bpf/verifier.c

12329

verbose(env, "%s not allowed in rbtree cb\n", func_name);

kernel/bpf/verifier.c

12334

ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta);

kernel/bpf/verifier.c

12349

verbose(env, "%s is %s ", reg_arg_name(env, argno),

kernel/bpf/verifier.c

12350

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

12351

verbose(env, "expected %s or socket\n",

kernel/bpf/verifier.c

12352

reg_type_str(env, base_type(reg->type) |

kernel/bpf/verifier.c

12356

ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno);

kernel/bpf/verifier.c

12363

verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",

kernel/bpf/verifier.c

12364

reg_arg_name(env, argno), btf_type_str(ref_t),

kernel/bpf/verifier.c

12368

ret = check_mem_reg(env, reg, argno, type_size);

kernel/bpf/verifier.c

1237

static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)

kernel/bpf/verifier.c

12381

ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg,

kernel/bpf/verifier.c

12384

verbose(env, "%s and ", reg_arg_name(env, argno));

kernel/bpf/verifier.c

12385

verbose(env, "%s memory, len pair leads to invalid memory access\n",

kernel/bpf/verifier.c

12386

reg_arg_name(env, next_argno));

kernel/bpf/verifier.c

12393

verifier_bug(env, "only one constant argument permitted");

kernel/bpf/verifier.c

12397

verbose(env, "%s must be a known constant\n",

kernel/bpf/verifier.c

12398

reg_arg_name(env, next_argno));

kernel/bpf/verifier.c

12411

verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

12418

verbose(env, "%s is neither owning or non-owning ref\n",

kernel/bpf/verifier.c

12419

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12427

verifier_bug(env, "Couldn't find btf_record");

kernel/bpf/verifier.c

12432

verbose(env, "%s doesn't point to a type with bpf_refcount field\n",

kernel/bpf/verifier.c

12433

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12442

verbose(env, "%s doesn't point to a const string\n",

kernel/bpf/verifier.c

12443

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12446

ret = check_arg_const_str(env, reg, argno);

kernel/bpf/verifier.c

12452

verbose(env, "%s doesn't point to a map value\n",

kernel/bpf/verifier.c

12453

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12456

ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map);

kernel/bpf/verifier.c

12462

verbose(env, "%s doesn't point to a map value\n",

kernel/bpf/verifier.c

12463

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12466

ret = process_timer_kfunc(env, reg, argno, meta);

kernel/bpf/verifier.c

12472

verbose(env, "%s doesn't point to a map value\n",

kernel/bpf/verifier.c

12473

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12476

ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map);

kernel/bpf/verifier.c

12482

verbose(env, "%s doesn't point to an irq flag on stack\n",

kernel/bpf/verifier.c

12483

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12486

ret = process_irq_flag(env, reg, argno, meta);

kernel/bpf/verifier.c

12495

verbose(env, "%s doesn't point to map value or allocated object\n",

kernel/bpf/verifier.c

12496

reg_arg_name(env, argno));

kernel/bpf/verifier.c

12508

ret = process_spin_lock(env, reg, argno, flags);

kernel/bpf/verifier.c

12519

int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

12527

err = fetch_kfunc_meta(env, func_id, offset, &kfunc);

kernel/bpf/verifier.c

12537

if (!kfunc.flags || !btf_kfunc_is_allowed(kfunc.btf, kfunc.id, env->prog))

kernel/bpf/verifier.c

12559

s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

12562

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

12567

if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)

kernel/bpf/verifier.c

12592

if (i < env->used_map_cnt) {

kernel/bpf/verifier.c

12593

size = is_key ? env->used_maps[i]->key_size

kernel/bpf/verifier.c

12594

: env->used_maps[i]->value_size;

kernel/bpf/verifier.c

12604

for (i = 0; i < env->used_map_cnt; i++) {

kernel/bpf/verifier.c

12605

struct bpf_map *map = env->used_maps[i];

kernel/bpf/verifier.c

12677

s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

12680

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

12688

if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0)

kernel/bpf/verifier.c

12742

static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,

kernel/bpf/verifier.c

12761

verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");

kernel/bpf/verifier.c

12765

ret_btf = env->prog->aux->btf;

kernel/bpf/verifier.c

12770

verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");

kernel/bpf/verifier.c

12776

verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");

kernel/bpf/verifier.c

12782

verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",

kernel/bpf/verifier.c

12811

if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {

kernel/bpf/verifier.c

12812

verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");

kernel/bpf/verifier.c

12817

verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");

kernel/bpf/verifier.c

12822

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12832

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12849

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12856

verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n",

kernel/bpf/verifier.c

12860

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12865

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12869

verbose(env,

kernel/bpf/verifier.c

12877

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12880

verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size");

kernel/bpf/verifier.c

12893

if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {

kernel/bpf/verifier.c

12894

verbose(env, "the prog does not allow writes to packet data\n");

kernel/bpf/verifier.c

12900

verifier_bug(env, "no dynptr id");

kernel/bpf/verifier.c

12911

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);

kernel/bpf/verifier.c

12913

static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

12917

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

12918

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

12933

err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);

kernel/bpf/verifier.c

12935

verbose(env, "calling kernel function %s is not allowed\n", meta.func_name);

kernel/bpf/verifier.c

12940

insn_aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

12950

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

12952

verbose(env, "failed to push state for failed lock acquisition\n");

kernel/bpf/verifier.c

12960

bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);

kernel/bpf/verifier.c

12962

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12963

err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1);

kernel/bpf/verifier.c

12965

verbose(env, "failed to mark s32 range for retval in forked state for lock\n");

kernel/bpf/verifier.c

12968

__mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32));

kernel/bpf/verifier.c

12970

verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n");

kernel/bpf/verifier.c

12975

verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");

kernel/bpf/verifier.c

12980

if (sleepable && !in_sleepable(env)) {

kernel/bpf/verifier.c

12981

verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);

kernel/bpf/verifier.c

12986

if (!in_sleepable_context(env))

kernel/bpf/verifier.c

12990

err = check_kfunc_args(env, &meta, insn_idx);

kernel/bpf/verifier.c

12997

(prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER

kernel/bpf/verifier.c

12998

&& !env->prog->sleepable))) {

kernel/bpf/verifier.c

13003

verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n",

kernel/bpf/verifier.c

13010

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

13013

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

13025

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

13028

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

13035

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

13038

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

13051

env->cur_state->active_rcu_locks++;

kernel/bpf/verifier.c

13053

if (env->cur_state->active_rcu_locks == 0) {

kernel/bpf/verifier.c

13054

verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);

kernel/bpf/verifier.c

13057

if (--env->cur_state->active_rcu_locks == 0)

kernel/bpf/verifier.c

13058

invalidate_rcu_protected_refs(env);

kernel/bpf/verifier.c

13060

env->cur_state->active_preempt_locks++;

kernel/bpf/verifier.c

13062

if (env->cur_state->active_preempt_locks == 0) {

kernel/bpf/verifier.c

13063

verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);

kernel/bpf/verifier.c

13066

env->cur_state->active_preempt_locks--;

kernel/bpf/verifier.c

13069

if (sleepable && !in_sleepable_context(env)) {

kernel/bpf/verifier.c

13070

verbose(env, "kernel func %s is sleepable within %s\n",

kernel/bpf/verifier.c

13071

func_name, non_sleepable_context_description(env));

kernel/bpf/verifier.c

13075

if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {

kernel/bpf/verifier.c

13076

verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");

kernel/bpf/verifier.c

13080

if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) {

kernel/bpf/verifier.c

13081

verbose(env, "kernel func %s requires RCU critical section protection\n", func_name);

kernel/bpf/verifier.c

13089

err = release_reg(env, &regs[meta.release_regno], false, !!meta.dynptr.id);

kernel/bpf/verifier.c

13098

ref_convert_owning_non_owning(env, id);

kernel/bpf/verifier.c

13103

verbose(env, "JIT does not support calling kfunc %s#%d\n",

kernel/bpf/verifier.c

13107

env->seen_exception = true;

kernel/bpf/verifier.c

13112

if (!env->exception_callback_subprog) {

kernel/bpf/verifier.c

13113

err = check_return_code(env, BPF_REG_1, "R1");

kernel/bpf/verifier.c

13122

bpf_mark_reg_not_init(env, &regs[regno]);

kernel/bpf/verifier.c

13125

invalidate_outgoing_stack_args(env, cur_func(env));

kernel/bpf/verifier.c

13135

verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");

kernel/bpf/verifier.c

13141

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

13144

__mark_reg_const_zero(env, &regs[BPF_REG_0]);

kernel/bpf/verifier.c

13145

mark_btf_func_reg_size(env, BPF_REG_0, t->size);

kernel/bpf/verifier.c

13148

err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);

kernel/bpf/verifier.c

13154

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

13167

verbose(env,

kernel/bpf/verifier.c

13175

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

13184

err = validate_ref_obj(env, &meta.ref_obj);

kernel/bpf/verifier.c

13199

(get_iter_from_state(env->cur_state, &meta)

kernel/bpf/verifier.c

13221

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

13230

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

13232

mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));

kernel/bpf/verifier.c

13234

id = acquire_reference(env, insn_idx, 0);

kernel/bpf/verifier.c

13239

ref_set_non_owning(env, &regs[BPF_REG_0]);

kernel/bpf/verifier.c

13243

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

13256

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

13260

struct bpf_func_state *caller = cur_func(env);

kernel/bpf/verifier.c

13261

struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno];

kernel/bpf/verifier.c

13275

mark_btf_func_reg_size(env, regno, sizeof(void *));

kernel/bpf/verifier.c

13278

mark_btf_func_reg_size(env, regno, t->size);

kernel/bpf/verifier.c

13282

err = process_iter_next_call(env, insn_idx, &meta);

kernel/bpf/verifier.c

13288

env->prog->call_session_cookie = true;

kernel/bpf/verifier.c

13291

return process_bpf_exit_full(env, NULL, true);

kernel/bpf/verifier.c

13296

static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13305

verbose(env, "math between %s pointer and %lld is not allowed\n",

kernel/bpf/verifier.c

13306

reg_type_str(env, type), val);

kernel/bpf/verifier.c

13311

verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",

kernel/bpf/verifier.c

13312

reg_type_str(env, type));

kernel/bpf/verifier.c

13317

verbose(env, "value %lld makes %s pointer be out of bounds\n",

kernel/bpf/verifier.c

13318

smin, reg_type_str(env, type));

kernel/bpf/verifier.c

13325

static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13334

verbose(env, "%s pointer offset %lld is not allowed\n",

kernel/bpf/verifier.c

13335

reg_type_str(env, type), val);

kernel/bpf/verifier.c

13340

verbose(env, "%s pointer offset %lld is not allowed\n",

kernel/bpf/verifier.c

13341

reg_type_str(env, type), smin);

kernel/bpf/verifier.c

13385

static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13388

return env->bypass_spec_v1 ||

kernel/bpf/verifier.c

13390

cur_aux(env)->nospec;

kernel/bpf/verifier.c

13410

static int sanitize_val_alu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13413

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

13415

if (can_skip_alu_sanitation(env, insn))

kernel/bpf/verifier.c

13431

static int sanitize_speculative_path(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13438

branch = push_stack(env, next_idx, curr_idx, true);

kernel/bpf/verifier.c

13442

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

13444

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

13445

mark_reg_unknown(env, regs, insn->src_reg);

kernel/bpf/verifier.c

13451

static int sanitize_ptr_alu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13459

struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;

kernel/bpf/verifier.c

13460

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

13469

if (can_skip_alu_sanitation(env, insn))

kernel/bpf/verifier.c

13508

env->explore_alu_limits = true;

kernel/bpf/verifier.c

13539

err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);

kernel/bpf/verifier.c

13547

static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

13549

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

13557

env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;

kernel/bpf/verifier.c

13560

static int sanitize_err(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13571

verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",

kernel/bpf/verifier.c

13575

verbose(env, "R%d has pointer with unsupported alu operation, %s\n",

kernel/bpf/verifier.c

13579

verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",

kernel/bpf/verifier.c

13583

verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",

kernel/bpf/verifier.c

13587

verbose(env, "R%d could not be pushed for speculative verification, %s\n",

kernel/bpf/verifier.c

13591

verifier_bug(env, "unknown reason (%d)", reason);

kernel/bpf/verifier.c

13606

struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13615

verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",

kernel/bpf/verifier.c

13621

verbose(env, "R%d stack pointer arithmetic goes out of range, "

kernel/bpf/verifier.c

13629

static int sanitize_check_bounds(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13638

if (env->bypass_spec_v1)

kernel/bpf/verifier.c

13643

if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,

kernel/bpf/verifier.c

13648

if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) {

kernel/bpf/verifier.c

13649

verbose(env, "R%d pointer arithmetic of map value goes out of range, "

kernel/bpf/verifier.c

13666

static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1367

static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)

kernel/bpf/verifier.c

13671

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

13689

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

13695

if (opcode == BPF_SUB && env->allow_ptr_leaks) {

kernel/bpf/verifier.c

13696

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

13700

verbose(env,

kernel/bpf/verifier.c

13707

verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",

kernel/bpf/verifier.c

13708

dst, reg_type_str(env, ptr_reg->type));

kernel/bpf/verifier.c

13743

verbose(env, "R%d pointer arithmetic on %s prohibited\n",

kernel/bpf/verifier.c

13744

dst, reg_type_str(env, ptr_reg->type));

kernel/bpf/verifier.c

13754

if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) ||

kernel/bpf/verifier.c

13755

!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type))

kernel/bpf/verifier.c

13762

ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,

kernel/bpf/verifier.c

13765

return sanitize_err(env, insn, ret, off_reg, dst_reg);

kernel/bpf/verifier.c

13783

dst_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

13797

verbose(env, "R%d tried to subtract pointer from scalar\n",

kernel/bpf/verifier.c

13806

verbose(env, "R%d subtraction from stack pointer prohibited\n",

kernel/bpf/verifier.c

13815

dst_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

13831

verbose(env, "R%d bitwise operator %s on pointer prohibited\n",

kernel/bpf/verifier.c

13836

verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",

kernel/bpf/verifier.c

13841

if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type))

kernel/bpf/verifier.c

13844

bounds_ret = sanitize_check_bounds(env, insn, dst_reg);

kernel/bpf/verifier.c

13848

ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,

kernel/bpf/verifier.c

1385

if (env->subprog_info[state->subprogno].stack_depth < size)

kernel/bpf/verifier.c

13850

if (verifier_bug_if(!can_skip_alu_sanitation(env, insn)

kernel/bpf/verifier.c

13851

&& !env->cur_state->speculative

kernel/bpf/verifier.c

13854

env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) {

kernel/bpf/verifier.c

13858

return sanitize_err(env, insn, ret, off_reg, dst_reg);

kernel/bpf/verifier.c

1386

env->subprog_info[state->subprogno].stack_depth = size;

kernel/bpf/verifier.c

1391

static int grow_stack_arg_slots(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1413

static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

1415

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

1427

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id)

kernel/bpf/verifier.c

1431

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

1435

s->id = ++env->id_gen;

kernel/bpf/verifier.c

1440

static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,

kernel/bpf/verifier.c

1443

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

1446

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

14559

static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

14573

branch = push_stack(env, env->insn_idx, env->insn_idx, false);

kernel/bpf/verifier.c

1459

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

14592

static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14603

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

14608

ret = sanitize_val_alu(env, insn);

kernel/bpf/verifier.c

1461

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

14610

return sanitize_err(env, insn, ret, NULL, NULL);

kernel/bpf/verifier.c

14639

env->fake_reg[0] = *dst_reg;

kernel/bpf/verifier.c

1464

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

14641

scalar32_min_max_sub(dst_reg, &env->fake_reg[0]);

kernel/bpf/verifier.c

14642

scalar_min_max_sub(dst_reg, &env->fake_reg[0]);

kernel/bpf/verifier.c

14643

dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off);

kernel/bpf/verifier.c

1468

s->id = ++env->id_gen;

kernel/bpf/verifier.c

14684

ret = maybe_fork_scalars(env, insn, dst_reg);

kernel/bpf/verifier.c

14694

ret = maybe_fork_scalars(env, insn, dst_reg);

kernel/bpf/verifier.c

14750

static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14753

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

14769

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

14774

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

14797

if (opcode == BPF_SUB && env->allow_ptr_leaks) {

kernel/bpf/verifier.c

14798

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

14801

verbose(env, "R%d pointer %s pointer prohibited\n",

kernel/bpf/verifier.c

14810

err = mark_chain_precision(env, insn->dst_reg);

kernel/bpf/verifier.c

14813

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

14818

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

14821

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

14825

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

14837

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

14843

print_verifier_state(env, vstate, vstate->curframe, true);

kernel/bpf/verifier.c

14844

verbose(env, "verifier internal error: unexpected ptr_reg\n");

kernel/bpf/verifier.c

14848

print_verifier_state(env, vstate, vstate->curframe, true);

kernel/bpf/verifier.c

14849

verbose(env, "verifier internal error: no src_reg\n");

kernel/bpf/verifier.c

14859

err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);

kernel/bpf/verifier.c

14871

if (env->bpf_capable &&

kernel/bpf/verifier.c

14918

static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

14920

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

14926

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

14930

if (is_pointer_value(env, insn->dst_reg)) {

kernel/bpf/verifier.c

14931

verbose(env, "R%d pointer arithmetic prohibited\n",

kernel/bpf/verifier.c

14938

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

14939

err = err ?: adjust_scalar_min_max_vals(env, insn,

kernel/bpf/verifier.c

14943

err = check_reg_arg(env, insn->dst_reg, DST_OP);

kernel/bpf/verifier.c

14952

if (!env->prog->aux->arena) {

kernel/bpf/verifier.c

14953

verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");

kernel/bpf/verifier.c

14959

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

14965

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

14976

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

14980

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

14986

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

14991

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

14992

verbose(env,

kernel/bpf/verifier.c

15001

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

15008

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

15013

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

15014

verbose(env,

kernel/bpf/verifier.c

15023

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

15031

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

15037

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

15041

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

15045

mark_reg_unknown(env, regs,

kernel/bpf/verifier.c

15056

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

1507

static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)

kernel/bpf/verifier.c

15071

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

15077

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

15083

verbose(env, "div by zero\n");

kernel/bpf/verifier.c

1509

return find_reference_state(env->cur_state, reg->id);

kernel/bpf/verifier.c

15092

verbose(env, "invalid shift %d\n", insn->imm);

kernel/bpf/verifier.c

15098

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

15099

err = err ?: adjust_reg_min_max_vals(env, insn);

kernel/bpf/verifier.c

15104

return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");

kernel/bpf/verifier.c

15193

static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32)

kernel/bpf/verifier.c

15196

regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32);

kernel/bpf/verifier.c

15197

reg_bounds_sync(&env->false_reg1);

kernel/bpf/verifier.c

15198

reg_bounds_sync(&env->false_reg2);

kernel/bpf/verifier.c

15204

if (range_bounds_violation(&env->false_reg1) || range_bounds_violation(&env->false_reg2))

kernel/bpf/verifier.c

15208

regs_refine_cond_op(&env->true_reg1, &env->true_reg2, opcode, is_jmp32);

kernel/bpf/verifier.c

15209

reg_bounds_sync(&env->true_reg1);

kernel/bpf/verifier.c

15210

reg_bounds_sync(&env->true_reg2);

kernel/bpf/verifier.c

15216

if (range_bounds_violation(&env->true_reg1) || range_bounds_violation(&env->true_reg2))

kernel/bpf/verifier.c

15226

static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1,

kernel/bpf/verifier.c

15379

return simulate_both_branches_taken(env, opcode, is_jmp32);

kernel/bpf/verifier.c

15450

static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1,

kernel/bpf/verifier.c

15468

if (!reg_not_null(env, reg1))

kernel/bpf/verifier.c

15489

return is_scalar_branch_taken(env, reg1, reg2, opcode, is_jmp32);

kernel/bpf/verifier.c

15656

static int regs_bounds_sanity_check_branches(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

15660

err = reg_bounds_sanity_check(env, &env->true_reg1, "true_reg1");

kernel/bpf/verifier.c

15661

err = err ?: reg_bounds_sanity_check(env, &env->true_reg2, "true_reg2");

kernel/bpf/verifier.c

15662

err = err ?: reg_bounds_sanity_check(env, &env->false_reg1, "false_reg1");

kernel/bpf/verifier.c

15663

err = err ?: reg_bounds_sanity_check(env, &env->false_reg2, "false_reg2");

kernel/bpf/verifier.c

15853

static void collect_linked_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

15858

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

15886

static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate,

kernel/bpf/verifier.c

15940

mark_reg_scratched(env, e->regno);

kernel/bpf/verifier.c

15942

mark_stack_slot_scratched(env, e->spi);

kernel/bpf/verifier.c

15946

static int check_cond_jmp_op(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

15949

struct bpf_verifier_state *this_branch = env->cur_state;

kernel/bpf/verifier.c

15963

verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);

kernel/bpf/verifier.c

15968

struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;

kernel/bpf/verifier.c

15971

prev_st = find_prev_entry(env, cur_st->parent, idx);

kernel/bpf/verifier.c

15974

queued_st = push_stack(env, idx + 1, idx, false);

kernel/bpf/verifier.c

15980

widen_imprecise_scalars(env, prev_st, queued_st);

kernel/bpf/verifier.c

15986

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

15993

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

15999

is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

16000

verbose(env, "R%d pointer comparison prohibited\n",

kernel/bpf/verifier.c

16010

src_reg = &env->fake_reg[0];

kernel/bpf/verifier.c

16020

err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0);

kernel/bpf/verifier.c

16026

env->false_reg1 = *dst_reg;

kernel/bpf/verifier.c

16027

env->false_reg2 = *src_reg;

kernel/bpf/verifier.c

16028

env->true_reg1 = *dst_reg;

kernel/bpf/verifier.c

16029

env->true_reg2 = *src_reg;

kernel/bpf/verifier.c

16030

pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32);

kernel/bpf/verifier.c

16036

err = mark_chain_precision(env, insn->dst_reg);

kernel/bpf/verifier.c

16039

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

16049

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

16050

err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx);

kernel/bpf/verifier.c

16054

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

16055

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

16063

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

16064

err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1,

kernel/bpf/verifier.c

16069

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

16070

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

16080

collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);

kernel/bpf/verifier.c

16082

collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);

kernel/bpf/verifier.c

16084

err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));

kernel/bpf/verifier.c

16089

other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);

kernel/bpf/verifier.c

16094

err = regs_bounds_sanity_check_branches(env);

kernel/bpf/verifier.c

16098

*dst_reg = env->false_reg1;

kernel/bpf/verifier.c

16099

*src_reg = env->false_reg2;

kernel/bpf/verifier.c

16100

other_branch_regs[insn->dst_reg] = env->true_reg1;

kernel/bpf/verifier.c

16102

other_branch_regs[insn->src_reg] = env->true_reg2;

kernel/bpf/verifier.c

16107

sync_linked_regs(env, this_branch, src_reg, &linked_regs);

kernel/bpf/verifier.c

16108

sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg],

kernel/bpf/verifier.c

16113

sync_linked_regs(env, this_branch, dst_reg, &linked_regs);

kernel/bpf/verifier.c

16114

sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg],

kernel/bpf/verifier.c

16174

is_pointer_value(env, insn->dst_reg)) {

kernel/bpf/verifier.c

16175

verbose(env, "R%d pointer comparison prohibited\n",

kernel/bpf/verifier.c

16179

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

16180

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

16185

static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

16187

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

16188

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

16194

verbose(env, "invalid BPF_LD_IMM insn\n");

kernel/bpf/verifier.c

16198

err = check_reg_arg(env, insn->dst_reg, DST_OP);

kernel/bpf/verifier.c

16215

mark_reg_known_zero(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

16228

verifier_bug(env, "pseudo btf id: unexpected dst reg type");

kernel/bpf/verifier.c

16235

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

16236

u32 subprogno = bpf_find_subprog(env,

kernel/bpf/verifier.c

16237

env->insn_idx + insn->imm + 1);

kernel/bpf/verifier.c

16240

verbose(env, "missing btf func_info\n");

kernel/bpf/verifier.c

16244

verbose(env, "callback function not static\n");

kernel/bpf/verifier.c

16253

map = env->used_maps[aux->map_index];

kernel/bpf/verifier.c

16258

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

16273

verifier_bug(env, "unexpected src reg value for ldimm64");

kernel/bpf/verifier.c

16307

static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

16309

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

16314

if (!may_access_skb(resolve_prog_type(env->prog))) {

kernel/bpf/verifier.c

16315

verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");

kernel/bpf/verifier.c

16319

if (!env->ops->gen_ld_abs) {

kernel/bpf/verifier.c

16320

verifier_bug(env, "gen_ld_abs is null");

kernel/bpf/verifier.c

16325

err = check_reg_arg(env, ctx_reg, SRC_OP);

kernel/bpf/verifier.c

16333

err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");

kernel/bpf/verifier.c

16338

verbose(env,

kernel/bpf/verifier.c

16345

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

16350

err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);

kernel/bpf/verifier.c

16356

bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);

kernel/bpf/verifier.c

16357

check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

16364

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

16366

regs[BPF_REG_0].subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

16371

if (env->cur_state->curframe) {

kernel/bpf/verifier.c

16374

mark_reg_scratched(env, BPF_REG_0);

kernel/bpf/verifier.c

16375

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

16378

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

16379

err = prepare_func_exit(env, &env->insn_idx);

kernel/bpf/verifier.c

16382

env->insn_idx--;

kernel/bpf/verifier.c

16388

static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range)

kernel/bpf/verifier.c

16390

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

16397

switch (env->prog->expected_attach_type) {

kernel/bpf/verifier.c

16418

if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS)

kernel/bpf/verifier.c

16428

if (!env->prog->aux->attach_btf_id)

kernel/bpf/verifier.c

16433

switch (env->prog->expected_attach_type) {

kernel/bpf/verifier.c

16451

switch (env->prog->expected_attach_type) {

kernel/bpf/verifier.c

16464

if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {

kernel/bpf/verifier.c

16466

if (!get_func_retval_range(env->prog, range))

kernel/bpf/verifier.c

16472

} else if (!env->prog->aux->attach_func_proto->type) {

kernel/bpf/verifier.c

16499

static bool program_returns_void(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

16501

const struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

16522

if (subprog_returns_void(env, 0))

kernel/bpf/verifier.c

16531

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)

kernel/bpf/verifier.c

16535

const struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

16536

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

16538

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

16539

struct bpf_func_state *frame = env->cur_state->frame[0];

kernel/bpf/verifier.c

16544

if (!frame->in_async_callback_fn && program_returns_void(env))

kernel/bpf/verifier.c

16556

if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg))

kernel/bpf/verifier.c

16557

return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);

kernel/bpf/verifier.c

16566

err = check_reg_arg(env, regno, SRC_OP);

kernel/bpf/verifier.c

16570

if (is_pointer_value(env, regno)) {

kernel/bpf/verifier.c

16571

verbose(env, "R%d leaks addr as return value\n", regno);

kernel/bpf/verifier.c

16584

if (prog_type == BPF_PROG_TYPE_CGROUP_SKB && (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS))

kernel/bpf/verifier.c

16587

if (!return_retval_range(env, &range))

kernel/bpf/verifier.c

16592

verbose(env, "%s the register R%d is not a known value (%s)\n",

kernel/bpf/verifier.c

16593

exit_ctx, regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

16597

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

16602

verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);

kernel/bpf/verifier.c

16606

verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");

kernel/bpf/verifier.c

16612

env->prog->enforce_expected_attach_type = 1;

kernel/bpf/verifier.c

16616

static int check_global_subprog_return_code(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

16618

struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);

kernel/bpf/verifier.c

16619

struct bpf_func_state *cur_frame = cur_func(env);

kernel/bpf/verifier.c

16622

if (subprog_returns_void(env, cur_frame->subprogno))

kernel/bpf/verifier.c

16625

err = check_reg_arg(env, BPF_REG_0, SRC_OP);

kernel/bpf/verifier.c

16630

if (is_arena_reg(env, BPF_REG_0))

kernel/bpf/verifier.c

16633

if (is_pointer_value(env, BPF_REG_0)) {

kernel/bpf/verifier.c

16634

verbose(env, "R%d leaks addr as return value\n", BPF_REG_0);

kernel/bpf/verifier.c

16639

verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",

kernel/bpf/verifier.c

16640

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

1665

static u32 state_htab_size(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

16654

bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)

kernel/bpf/verifier.c

16663

return env->prog->jit_requested && bpf_jit_supports_percpu_insn();

kernel/bpf/verifier.c

1667

return env->prog->len;

kernel/bpf/verifier.c

16673

bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,

kernel/bpf/verifier.c

16682

if (bpf_get_helper_proto(env, call->imm, &fn) < 0)

kernel/bpf/verifier.c

16686

(bpf_verifier_inlines_helper_call(env, call->imm) ||

kernel/bpf/verifier.c

1670

struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

16701

err = bpf_fetch_kfunc_arg_meta(env, call->imm, call->off, &meta);

kernel/bpf/verifier.c

1672

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

1675

return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];

kernel/bpf/verifier.c

16788

static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

16792

struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;

kernel/bpf/verifier.c

16793

struct bpf_insn *call = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

16800

if (!bpf_get_call_summary(env, call, &cs))

kernel/bpf/verifier.c

16823

if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)

kernel/bpf/verifier.c

16846

env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;

kernel/bpf/verifier.c

16847

env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;

kernel/bpf/verifier.c

16869

env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;

kernel/bpf/verifier.c

16875

static int mark_fastcall_patterns(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

16877

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

16882

for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {

kernel/bpf/verifier.c

16886

insn = env->prog->insnsi + i;

kernel/bpf/verifier.c

16894

insn = env->prog->insnsi + i;

kernel/bpf/verifier.c

16897

mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);

kernel/bpf/verifier.c

16903

static void adjust_btf_func(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

16905

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

16912

for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)

kernel/bpf/verifier.c

16913

aux->func_info[i].insn_off = env->subprog_info[i].start;

kernel/bpf/verifier.c

16952

void bpf_clear_singular_ids(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

16955

struct bpf_idset *idset = &env->idset_scratch;

kernel/bpf/verifier.c

17030

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,

kernel/bpf/verifier.c

17033

enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;

kernel/bpf/verifier.c

1705

static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,

kernel/bpf/verifier.c

17070

verbose(env, "same insn cannot be used with different pointers\n");

kernel/bpf/verifier.c

1708

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

17083

static int process_bpf_exit_full(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17087

struct bpf_func_state *cur_frame = cur_func(env);

kernel/bpf/verifier.c

1709

struct bpf_verifier_stack_elem *elem, *head = env->head;

kernel/bpf/verifier.c

17095

int err = check_resource_leak(env, exception_exit,

kernel/bpf/verifier.c

17096

exception_exit || !env->cur_state->curframe,

kernel/bpf/verifier.c

17114

if (env->cur_state->curframe) {

kernel/bpf/verifier.c

17116

err = prepare_func_exit(env, &env->insn_idx);

kernel/bpf/verifier.c

1712

if (env->head == NULL)

kernel/bpf/verifier.c

17135

err = check_global_subprog_return_code(env);

kernel/bpf/verifier.c

17137

err = check_return_code(env, BPF_REG_0, "R0");

kernel/bpf/verifier.c

17143

static int indirect_jump_min_max_index(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17148

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

17154

verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg));

kernel/bpf/verifier.c

17158

verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg));

kernel/bpf/verifier.c

17166

verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n",

kernel/bpf/verifier.c

17177

static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

17187

dst_reg = reg_state(env, insn->dst_reg);

kernel/bpf/verifier.c

17189

verbose(env, "R%d has type %s, expected PTR_TO_INSN\n",

kernel/bpf/verifier.c

17190

insn->dst_reg, reg_type_str(env, dst_reg->type));

kernel/bpf/verifier.c

17195

if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg))

kernel/bpf/verifier.c

17198

if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env,

kernel/bpf/verifier.c

17202

err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index);

kernel/bpf/verifier.c

17207

if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) {

kernel/bpf/verifier.c

17208

env->gotox_tmp_buf = bpf_iarray_realloc(env->gotox_tmp_buf,

kernel/bpf/verifier.c

1721

bpf_vlog_reset(&env->log, head->log_pos);

kernel/bpf/verifier.c

17210

if (!env->gotox_tmp_buf)

kernel/bpf/verifier.c

17214

n = bpf_copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items);

kernel/bpf/verifier.c

17218

verbose(env, "register R%d doesn't point to any offset in map id=%d\n",

kernel/bpf/verifier.c

17224

mark_indirect_target(env, env->gotox_tmp_buf->items[i]);

kernel/bpf/verifier.c

17225

other_branch = push_stack(env, env->gotox_tmp_buf->items[i],

kernel/bpf/verifier.c

17226

env->insn_idx, env->cur_state->speculative);

kernel/bpf/verifier.c

17230

env->insn_idx = env->gotox_tmp_buf->items[n-1];

kernel/bpf/verifier.c

17231

mark_indirect_target(env, env->insn_idx);

kernel/bpf/verifier.c

17235

static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)

kernel/bpf/verifier.c

17238

struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];

kernel/bpf/verifier.c

17244

return check_alu_op(env, insn);

kernel/bpf/verifier.c

17247

return check_load_mem(env, insn, false,

kernel/bpf/verifier.c

17253

return check_atomic(env, insn);

kernel/bpf/verifier.c

17254

return check_store_reg(env, insn, false);

kernel/bpf/verifier.c

17259

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

17262

return check_stack_arg_write(env, state, insn->off, NULL);

kernel/bpf/verifier.c

17267

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

17271

dst_reg_type = cur_regs(env)[insn->dst_reg].type;

kernel/bpf/verifier.c

17273

err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg),

kernel/bpf/verifier.c

17279

return save_aux_ptr_type(env, dst_reg_type, false);

kernel/bpf/verifier.c

17285

env->jmps_processed++;

kernel/bpf/verifier.c

17287

if (env->cur_state->active_locks) {

kernel/bpf/verifier.c

1729

env->head = elem;

kernel/bpf/verifier.c

17293

verbose(env,

kernel/bpf/verifier.c

17298

mark_reg_scratched(env, BPF_REG_0);

kernel/bpf/verifier.c

17299

if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno]))

kernel/bpf/verifier.c

1730

env->stack_size--;

kernel/bpf/verifier.c

17300

cur_func(env)->no_stack_arg_load = true;

kernel/bpf/verifier.c

17302

return check_func_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

17304

return check_kfunc_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

17305

return check_helper_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

17308

return check_indirect_jump(env, insn);

kernel/bpf/verifier.c

17311

env->insn_idx += insn->off + 1;

kernel/bpf/verifier.c

17313

env->insn_idx += insn->imm + 1;

kernel/bpf/verifier.c

17316

return process_bpf_exit_full(env, do_print_state, false);

kernel/bpf/verifier.c

17318

return check_cond_jmp_op(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

17324

return check_ld_abs(env, insn);

kernel/bpf/verifier.c

17327

err = check_ld_imm(env, insn);

kernel/bpf/verifier.c

17331

env->insn_idx++;

kernel/bpf/verifier.c

17332

sanitize_mark_insn_seen(env);

kernel/bpf/verifier.c

17341

static int do_check(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

17343

bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);

kernel/bpf/verifier.c

17344

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

17345

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/verifier.c

17346

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

17356

env->cur_hist_ent = NULL;

kernel/bpf/verifier.c

17358

env->prev_insn_idx = prev_insn_idx;

kernel/bpf/verifier.c

17359

if (env->insn_idx >= insn_cnt) {

kernel/bpf/verifier.c

17360

verbose(env, "invalid insn idx %d insn_cnt %d\n",

kernel/bpf/verifier.c

17361

env->insn_idx, insn_cnt);

kernel/bpf/verifier.c

17365

insn = &insns[env->insn_idx];

kernel/bpf/verifier.c

17366

insn_aux = &env->insn_aux_data[env->insn_idx];

kernel/bpf/verifier.c

17368

if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {

kernel/bpf/verifier.c

17369

verbose(env,

kernel/bpf/verifier.c

17371

env->insn_processed);

kernel/bpf/verifier.c

17375

state->last_insn_idx = env->prev_insn_idx;

kernel/bpf/verifier.c

17376

state->insn_idx = env->insn_idx;

kernel/bpf/verifier.c

17378

if (bpf_is_prune_point(env, env->insn_idx)) {

kernel/bpf/verifier.c

17379

err = bpf_is_state_visited(env, env->insn_idx);

kernel/bpf/verifier.c

17384

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

17386

verbose(env, "\nfrom %d to %d%s: safe\n",

kernel/bpf/verifier.c

17387

env->prev_insn_idx, env->insn_idx,

kernel/bpf/verifier.c

17388

env->cur_state->speculative ?

kernel/bpf/verifier.c

17391

verbose(env, "%d: safe\n", env->insn_idx);

kernel/bpf/verifier.c

17397

if (bpf_is_jmp_point(env, env->insn_idx)) {

kernel/bpf/verifier.c

17398

err = bpf_push_jmp_history(env, state, 0, 0, 0, 0);

kernel/bpf/verifier.c

17409

if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {

kernel/bpf/verifier.c

17410

verbose(env, "\nfrom %d to %d%s:",

kernel/bpf/verifier.c

17411

env->prev_insn_idx, env->insn_idx,

kernel/bpf/verifier.c

17412

env->cur_state->speculative ?

kernel/bpf/verifier.c

17414

print_verifier_state(env, state, state->curframe, true);

kernel/bpf/verifier.c

17418

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

17419

if (verifier_state_scratched(env))

kernel/bpf/verifier.c

17420

print_insn_state(env, state, state->curframe);

kernel/bpf/verifier.c

17422

verbose_linfo(env, env->insn_idx, "; ");

kernel/bpf/verifier.c

17423

env->prev_log_pos = env->log.end_pos;

kernel/bpf/verifier.c

17424

verbose(env, "%d: ", env->insn_idx);

kernel/bpf/verifier.c

17425

bpf_verbose_insn(env, insn);

kernel/bpf/verifier.c

17426

env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;

kernel/bpf/verifier.c

17427

env->prev_log_pos = env->log.end_pos;

kernel/bpf/verifier.c

17430

if (bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

17431

err = bpf_prog_offload_verify_insn(env, env->insn_idx,

kernel/bpf/verifier.c

17432

env->prev_insn_idx);

kernel/bpf/verifier.c

17437

sanitize_mark_insn_seen(env);

kernel/bpf/verifier.c

17438

prev_insn_idx = env->insn_idx;

kernel/bpf/verifier.c

17442

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

17455

env, "const R%d: %u != %llu",

kernel/bpf/verifier.c

1746

static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17467

err = do_check_insn(env, &do_print_state);

kernel/bpf/verifier.c

17484

env->insn_idx++;

kernel/bpf/verifier.c

1750

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

17504

BPF_OP(insn->code) != BPF_CALL, env,

kernel/bpf/verifier.c

17508

mark_verifier_state_scratched(env);

kernel/bpf/verifier.c

17509

err = bpf_update_branch_counts(env, env->cur_state);

kernel/bpf/verifier.c

17512

err = pop_stack(env, &prev_insn_idx, &env->insn_idx,

kernel/bpf/verifier.c

17559

static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)

kernel/bpf/verifier.c

17566

for (i = 0; i < env->used_btf_cnt; i++)

kernel/bpf/verifier.c

17567

if (env->used_btfs[i].btf == btf)

kernel/bpf/verifier.c

17570

if (env->used_btf_cnt >= MAX_USED_BTFS) {

kernel/bpf/verifier.c

17571

verbose(env, "The total number of btfs per program has reached the limit of %u\n",

kernel/bpf/verifier.c

17577

btf_mod = &env->used_btfs[env->used_btf_cnt];

kernel/bpf/verifier.c

17590

env->used_btf_cnt++;

kernel/bpf/verifier.c

1760

elem->next = env->head;

kernel/bpf/verifier.c

17600

static int __check_pseudo_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1761

elem->log_pos = env->log.end_pos;

kernel/bpf/verifier.c

17617

verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);

kernel/bpf/verifier.c

1762

env->head = elem;

kernel/bpf/verifier.c

17622

verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);

kernel/bpf/verifier.c

17629

verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",

kernel/bpf/verifier.c

1763

env->stack_size++;

kernel/bpf/verifier.c

17668

verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",

kernel/bpf/verifier.c

1768

if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {

kernel/bpf/verifier.c

17683

static int check_pseudo_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1769

verbose(env, "The sequence of %d jumps is too complex.\n",

kernel/bpf/verifier.c

17695

verbose(env, "invalid module BTF object FD specified.\n");

kernel/bpf/verifier.c

1770

env->stack_size);

kernel/bpf/verifier.c

17700

verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");

kernel/bpf/verifier.c

17707

err = __check_pseudo_btf_id(env, insn, aux, btf);

kernel/bpf/verifier.c

17713

return __add_used_btf(env, btf);

kernel/bpf/verifier.c

17736

static int check_map_prog_compatibility(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17745

verbose(env, "program's hash doesn't match map's excl_prog_hash\n");

kernel/bpf/verifier.c

17752

verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");

kernel/bpf/verifier.c

17759

verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");

kernel/bpf/verifier.c

17764

verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");

kernel/bpf/verifier.c

17771

verbose(env, "offload device mismatch between prog and map\n");

kernel/bpf/verifier.c

17776

verbose(env, "bpf_struct_ops map cannot be used in prog\n");

kernel/bpf/verifier.c

17805

verbose(env,

kernel/bpf/verifier.c

17811

bpf_cgroup_storage_assign(env->prog->aux, map)) {

kernel/bpf/verifier.c

17812

verbose(env, "only one cgroup storage of each type is allowed\n");

kernel/bpf/verifier.c

17817

if (env->prog->aux->arena) {

kernel/bpf/verifier.c

17818

verbose(env, "Only one arena per program\n");

kernel/bpf/verifier.c

17821

if (!env->allow_ptr_leaks || !env->bpf_capable) {

kernel/bpf/verifier.c

17822

verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");

kernel/bpf/verifier.c

17825

if (!env->prog->jit_requested) {

kernel/bpf/verifier.c

17826

verbose(env, "JIT is required to use arena\n");

kernel/bpf/verifier.c

17830

verbose(env, "JIT doesn't support arena\n");

kernel/bpf/verifier.c

17833

env->prog->aux->arena = (void *)map;

kernel/bpf/verifier.c

17834

if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {

kernel/bpf/verifier.c

17835

verbose(env, "arena's user address must be set via map_extra or mmap()\n");

kernel/bpf/verifier.c

17843

static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)

kernel/bpf/verifier.c

17848

for (i = 0; i < env->used_map_cnt; i++)

kernel/bpf/verifier.c

17849

if (env->used_maps[i] == map)

kernel/bpf/verifier.c

17852

if (env->used_map_cnt >= MAX_USED_MAPS) {

kernel/bpf/verifier.c

17853

verbose(env, "The total number of maps per program has reached the limit of %u\n",

kernel/bpf/verifier.c

17858

err = check_map_prog_compatibility(env, map, env->prog);

kernel/bpf/verifier.c

17862

if (env->prog->sleepable)

kernel/bpf/verifier.c

17872

env->used_maps[env->used_map_cnt++] = map;

kernel/bpf/verifier.c

17875

err = bpf_insn_array_init(map, env->prog);

kernel/bpf/verifier.c

17877

verbose(env, "Failed to properly initialize insn array\n");

kernel/bpf/verifier.c

1788

static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno)

kernel/bpf/verifier.c

17880

env->insn_array_maps[env->insn_array_map_cnt++] = map;

kernel/bpf/verifier.c

17883

return env->used_map_cnt - 1;

kernel/bpf/verifier.c

17890

static int add_used_map(struct bpf_verifier_env *env, int fd)

kernel/bpf/verifier.c

17897

verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);

kernel/bpf/verifier.c

1790

char *buf = env->tmp_arg_name;

kernel/bpf/verifier.c

17901

return __add_used_map(env, map);

kernel/bpf/verifier.c

17904

static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

1791

int len = sizeof(env->tmp_arg_name);

kernel/bpf/verifier.c

17913

verbose(env, "BPF_NEG uses reserved fields\n");

kernel/bpf/verifier.c

17921

verbose(env, "BPF_END uses reserved fields\n");

kernel/bpf/verifier.c

17930

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

17935

verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");

kernel/bpf/verifier.c

17940

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

17944

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

17962

verbose(env, "BPF_ALU uses reserved fields\n");

kernel/bpf/verifier.c

17968

verbose(env, "BPF_ALU uses reserved fields\n");

kernel/bpf/verifier.c

17973

verbose(env, "invalid BPF_ALU opcode %x\n", opcode);

kernel/bpf/verifier.c

17978

static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

17990

verbose(env, "BPF_CALL uses reserved fields\n");

kernel/bpf/verifier.c

17997

verbose(env, "BPF_JA|BPF_X uses reserved fields\n");

kernel/bpf/verifier.c

18003

verbose(env, "BPF_JA uses reserved fields\n");

kernel/bpf/verifier.c

18011

verbose(env, "BPF_EXIT uses reserved fields\n");

kernel/bpf/verifier.c

18018

verbose(env, "invalid may_goto imm %d\n", insn->imm);

kernel/bpf/verifier.c

18025

verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");

kernel/bpf/verifier.c

18029

verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");

kernel/bpf/verifier.c

18036

static int check_insn_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

18041

return check_alu_fields(env, insn);

kernel/bpf/verifier.c

18045

verbose(env, "BPF_LDX uses reserved fields\n");

kernel/bpf/verifier.c

18053

verbose(env, "BPF_STX uses reserved fields\n");

kernel/bpf/verifier.c

18059

verbose(env, "BPF_ST uses reserved fields\n");

kernel/bpf/verifier.c

18065

return check_jmp_fields(env, insn);

kernel/bpf/verifier.c

18073

verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");

kernel/bpf/verifier.c

18077

verbose(env, "invalid BPF_LD mode\n");

kernel/bpf/verifier.c

18083

verbose(env, "unknown insn class %d\n", BPF_CLASS(insn->code));

kernel/bpf/verifier.c

18096

static int check_and_resolve_insns(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18098

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

18099

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

18102

err = bpf_prog_calc_tag(env->prog);

kernel/bpf/verifier.c

18109

verbose(env, "R%d is invalid\n", insn->dst_reg);

kernel/bpf/verifier.c

18113

verbose(env, "R%d is invalid\n", insn->src_reg);

kernel/bpf/verifier.c

18126

verbose(env, "invalid bpf_ld_imm64 insn\n");

kernel/bpf/verifier.c

18131

verbose(env, "BPF_LD_IMM64 uses reserved fields\n");

kernel/bpf/verifier.c

18140

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

18141

err = check_pseudo_btf_id(env, insn, aux);

kernel/bpf/verifier.c

18148

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

18166

verbose(env, "unrecognized bpf_ld_imm64 insn\n");

kernel/bpf/verifier.c

18173

if (bpfptr_is_null(env->fd_array)) {

kernel/bpf/verifier.c

18174

verbose(env, "fd_idx without fd_array is invalid\n");

kernel/bpf/verifier.c

18177

if (copy_from_bpfptr_offset(&fd, env->fd_array,

kernel/bpf/verifier.c

18187

map_idx = add_used_map(env, fd);

kernel/bpf/verifier.c

18190

map = env->used_maps[map_idx];

kernel/bpf/verifier.c

18192

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

18202

verbose(env, "no direct value access support for this map type\n");

kernel/bpf/verifier.c

18208

verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",

kernel/bpf/verifier.c

18228

verbose(env, "unknown opcode %02x\n", insn->code);

kernel/bpf/verifier.c

18232

err = check_insn_fields(env, insn);

kernel/bpf/verifier.c

18245

static void release_maps(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18247

__bpf_free_used_maps(env->prog->aux, env->used_maps,

kernel/bpf/verifier.c

18248

env->used_map_cnt);

kernel/bpf/verifier.c

18252

static void release_btfs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18254

__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);

kernel/bpf/verifier.c

18258

static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18260

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

18261

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

18273

static void release_insn_arrays(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18277

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/verifier.c

18278

bpf_insn_array_release(env->insn_array_maps[i]);

kernel/bpf/verifier.c

18294

static void sanitize_dead_code(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18296

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

18298

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

18299

const int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

18312

static void free_states(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18319

bpf_free_verifier_state(env->cur_state, true);

kernel/bpf/verifier.c

18320

env->cur_state = NULL;

kernel/bpf/verifier.c

18321

while (!pop_stack(env, NULL, NULL, false));

kernel/bpf/verifier.c

18323

list_for_each_safe(pos, tmp, &env->free_list) {

kernel/bpf/verifier.c

18328

INIT_LIST_HEAD(&env->free_list);

kernel/bpf/verifier.c

18330

for (i = 0; i < env->scc_cnt; ++i) {

kernel/bpf/verifier.c

18331

info = env->scc_info[i];

kernel/bpf/verifier.c

18337

env->scc_info[i] = NULL;

kernel/bpf/verifier.c

18340

if (!env->explored_states)

kernel/bpf/verifier.c

18343

for (i = 0; i < state_htab_size(env); i++) {

kernel/bpf/verifier.c

18344

head = &env->explored_states[i];

kernel/bpf/verifier.c

18351

INIT_LIST_HEAD(&env->explored_states[i]);

kernel/bpf/verifier.c

18355

static int do_check_common(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

18357

bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);

kernel/bpf/verifier.c

18358

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

18359

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

18364

env->prev_linfo = NULL;

kernel/bpf/verifier.c

18365

env->pass_cnt++;

kernel/bpf/verifier.c

18373

state->in_sleepable = env->prog->sleepable;

kernel/bpf/verifier.c

18379

env->cur_state = state;

kernel/bpf/verifier.c

18380

init_func_state(env, state->frame[0],

kernel/bpf/verifier.c

18384

state->first_insn_idx = env->subprog_info[subprog].start;

kernel/bpf/verifier.c

18388

if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {

kernel/bpf/verifier.c

18389

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

18393

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

18394

verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);

kernel/bpf/verifier.c

18395

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

18399

if (subprog_is_exc_cb(env, subprog)) {

kernel/bpf/verifier.c

18406

if (subprog_returns_void(env, subprog)) {

kernel/bpf/verifier.c

18407

verbose(env, "exception cb cannot return void\n");

kernel/bpf/verifier.c

18414

verbose(env, "exception cb only supports single integer argument\n");

kernel/bpf/verifier.c

18425

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

18428

mark_reg_unknown(env, regs, i);

kernel/bpf/verifier.c

1843

static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

18431

__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0);

kernel/bpf/verifier.c

18436

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

18439

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

18448

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

18451

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

18454

mark_reg_unknown(env, regs, i);

kernel/bpf/verifier.c

18456

verifier_bug(env, "unhandled arg#%d type %d",

kernel/bpf/verifier.c

18462

if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) {

kernel/bpf/verifier.c

18463

verbose(env, "freplace programs with >%d args not supported yet\n",

kernel/bpf/verifier.c

18473

if (env->prog->aux->func_info_aux) {

kernel/bpf/verifier.c

18474

ret = btf_prepare_func_args(env, 0);

kernel/bpf/verifier.c

18476

env->prog->aux->func_info_aux[0].unreliable = true;

kernel/bpf/verifier.c

18484

mark_reg_known_zero(env, regs, BPF_REG_1);

kernel/bpf/verifier.c

18488

if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {

kernel/bpf/verifier.c

18490

ret = aux->ctx_arg_info[i].refcounted ? acquire_reference(env, 0, 0) : 0;

kernel/bpf/verifier.c

18498

ret = do_check(env);

kernel/bpf/verifier.c

1850

reg->precise = !env->bpf_capable;

kernel/bpf/verifier.c

18501

bpf_vlog_reset(&env->log, 0);

kernel/bpf/verifier.c

18502

free_states(env);

kernel/bpf/verifier.c

18526

static int do_check_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18528

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

1853

static void mark_reg_known_zero(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18537

if (env->exception_callback_subprog)

kernel/bpf/verifier.c

18538

subprog_aux(env, env->exception_callback_subprog)->called = true;

kernel/bpf/verifier.c

18542

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

18543

if (!bpf_subprog_is_global(env, i))

kernel/bpf/verifier.c

18546

insn_processed = env->insn_processed;

kernel/bpf/verifier.c

18548

sub_aux = subprog_aux(env, i);

kernel/bpf/verifier.c

18552

env->insn_idx = env->subprog_info[i].start;

kernel/bpf/verifier.c

18553

WARN_ON_ONCE(env->insn_idx == 0);

kernel/bpf/verifier.c

18554

ret = do_check_common(env, i);

kernel/bpf/verifier.c

18555

env->subprog_info[i].insn_processed = env->insn_processed - insn_processed;

kernel/bpf/verifier.c

18558

} else if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

18559

verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",

kernel/bpf/verifier.c

18560

i, subprog_name(env, i));

kernel/bpf/verifier.c

18580

static int do_check_main(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18582

u32 insn_processed = env->insn_processed;

kernel/bpf/verifier.c

18585

env->insn_idx = 0;

kernel/bpf/verifier.c

18586

ret = do_check_common(env, 0);

kernel/bpf/verifier.c

18587

env->subprog_info[0].insn_processed = env->insn_processed - insn_processed;

kernel/bpf/verifier.c

18589

env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;

kernel/bpf/verifier.c

18594

static void print_verification_stats(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18597

int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt;

kernel/bpf/verifier.c

18599

if (env->log.level & BPF_LOG_STATS) {

kernel/bpf/verifier.c

18600

verbose(env, "verification time %lld usec\n",

kernel/bpf/verifier.c

18601

div_u64(env->verification_time, 1000));

kernel/bpf/verifier.c

18602

verbose(env, "stack depth %d", env->subprog_info[0].stack_depth);

kernel/bpf/verifier.c

18604

verbose(env, "+%d", env->subprog_info[i].stack_depth);

kernel/bpf/verifier.c

18605

verbose(env, " max %d\n", env->max_stack_depth);

kernel/bpf/verifier.c

18606

verbose(env, "insns processed %d", env->subprog_info[0].insn_processed);

kernel/bpf/verifier.c

18608

if (bpf_subprog_is_global(env, i))

kernel/bpf/verifier.c

18609

verbose(env, "+%d", env->subprog_info[i].insn_processed);

kernel/bpf/verifier.c

18610

verbose(env, "\n");

kernel/bpf/verifier.c

18612

verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "

kernel/bpf/verifier.c

18614

env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,

kernel/bpf/verifier.c

18615

env->max_states_per_insn, env->total_states,

kernel/bpf/verifier.c

18616

env->peak_states, env->longest_mark_read_walk);

kernel/bpf/verifier.c

18628

static int check_struct_ops_btf_id(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18634

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

18642

verbose(env, "struct ops programs must have a GPL compatible license\n");

kernel/bpf/verifier.c

18652

env->attach_btf_mod = btf_try_get_module(btf);

kernel/bpf/verifier.c

18653

if (!env->attach_btf_mod) {

kernel/bpf/verifier.c

18654

verbose(env, "struct_ops module %s is not found\n",

kernel/bpf/verifier.c

18663

verbose(env, "attach_btf_id %u is not a supported struct\n",

kernel/bpf/verifier.c

18672

verbose(env, "attach to invalid member idx %u of struct %s\n",

kernel/bpf/verifier.c

18682

verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",

kernel/bpf/verifier.c

18690

verbose(env, "attach to unsupported member %s of struct %s\n",

kernel/bpf/verifier.c

18699

verbose(env, "attach to unsupported member %s of struct %s\n",

kernel/bpf/verifier.c

18706

verbose(env, "Private stack not supported by jit\n");

kernel/bpf/verifier.c

18720

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

18721

if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {

kernel/bpf/verifier.c

18722

verbose(env, "program with __ref argument cannot tail call\n");

kernel/bpf/verifier.c

18732

env->ops = st_ops->verifier_ops;

kernel/bpf/verifier.c

19331

static int check_attach_btf_id(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

19333

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

19345

verbose(env, "Syscall programs can only be sleepable\n");

kernel/bpf/verifier.c

19350

verbose(env, "Program of this type cannot be sleepable\n");

kernel/bpf/verifier.c

19355

return check_struct_ops_btf_id(env);

kernel/bpf/verifier.c

19362

ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);

kernel/bpf/verifier.c

19371

env->ops = bpf_verifier_ops[tgt_prog->type];

kernel/bpf/verifier.c

19393

ret = bpf_lsm_verify_prog(&env->log, prog);

kernel/bpf/verifier.c

19398

verbose(env, "Attaching tracing programs to function '%s' is rejected.\n",

kernel/bpf/verifier.c

19406

verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n",

kernel/bpf/verifier.c

19501

static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)

kernel/bpf/verifier.c

19510

err = __add_used_map(env, map);

kernel/bpf/verifier.c

19519

return __add_used_btf(env, btf);

kernel/bpf/verifier.c

19522

verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);

kernel/bpf/verifier.c

19526

static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)

kernel/bpf/verifier.c

19533

env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);

kernel/bpf/verifier.c

19545

verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);

kernel/bpf/verifier.c

19550

if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))

kernel/bpf/verifier.c

19553

ret = add_fd_from_fd_array(env, fd);

kernel/bpf/verifier.c

19562

static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx)

kernel/bpf/verifier.c

19564

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

19581

seen_direct_write = env->seen_direct_write;

kernel/bpf/verifier.c

19582

is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);

kernel/bpf/verifier.c

19590

env->seen_direct_write = seen_direct_write;

kernel/bpf/verifier.c

19598

if (!env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

19601

if (env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

19604

if (env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

19628

int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

19635

verbose(env, "invalid kernel function call not eliminated in verifier pass\n");

kernel/bpf/verifier.c

19645

desc = find_kfunc_desc(env->prog, insn->imm, insn->off);

kernel/bpf/verifier.c

19647

verifier_bug(env, "kernel function descriptor not found for func_id %u",

kernel/bpf/verifier.c

19652

err = specialize_kfunc(env, desc, insn_idx);

kernel/bpf/verifier.c

19660

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

19662

u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;

kernel/bpf/verifier.c

19665

verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

19678

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

19682

verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

19688

verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

19699

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

19713

verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

19718

__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,

kernel/bpf/verifier.c

19725

(env->prog->expected_attach_type == BPF_TRACE_FSESSION ||

kernel/bpf/verifier.c

19726

env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {

kernel/bpf/verifier.c

19740

(env->prog->expected_attach_type == BPF_TRACE_FSESSION ||

kernel/bpf/verifier.c

19741

env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {

kernel/bpf/verifier.c

19759

if (env->insn_aux_data[insn_idx].arg_prog) {

kernel/bpf/verifier.c

19760

u32 regno = env->insn_aux_data[insn_idx].arg_prog;

kernel/bpf/verifier.c

19761

struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };

kernel/bpf/verifier.c

19776

struct bpf_verifier_env *env;

kernel/bpf/verifier.c

19789

env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

19790

if (!env)

kernel/bpf/verifier.c

19793

env->bt.env = env;

kernel/bpf/verifier.c

19796

env->insn_aux_data =

kernel/bpf/verifier.c

19799

if (!env->insn_aux_data)

kernel/bpf/verifier.c

19802

env->insn_aux_data[i].orig_idx = i;

kernel/bpf/verifier.c

19803

env->succ = bpf_iarray_realloc(NULL, 2);

kernel/bpf/verifier.c

19804

if (!env->succ)

kernel/bpf/verifier.c

19806

env->prog = *prog;

kernel/bpf/verifier.c

19807

env->ops = bpf_verifier_ops[env->prog->type];

kernel/bpf/verifier.c

19809

env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);

kernel/bpf/verifier.c

19810

env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);

kernel/bpf/verifier.c

19811

env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);

kernel/bpf/verifier.c

19812

env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);

kernel/bpf/verifier.c

19813

env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);

kernel/bpf/verifier.c

19824

ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);

kernel/bpf/verifier.c

19828

ret = process_fd_array(env, attr, uattr);

kernel/bpf/verifier.c

19832

mark_verifier_state_clean(env);

kernel/bpf/verifier.c

19836

verbose(env, "in-kernel BTF is malformed\n");

kernel/bpf/verifier.c

19841

env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);

kernel/bpf/verifier.c

19843

env->strict_alignment = true;

kernel/bpf/verifier.c

19845

env->strict_alignment = false;

kernel/bpf/verifier.c

19848

env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;

kernel/bpf/verifier.c

19849

env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;

kernel/bpf/verifier.c

19851

env->explored_states = kvzalloc_objs(struct list_head,

kernel/bpf/verifier.c

19852

state_htab_size(env),

kernel/bpf/verifier.c

19855

if (!env->explored_states)

kernel/bpf/verifier.c

19858

for (i = 0; i < state_htab_size(env); i++)

kernel/bpf/verifier.c

19859

INIT_LIST_HEAD(&env->explored_states[i]);

kernel/bpf/verifier.c

19860

INIT_LIST_HEAD(&env->free_list);

kernel/bpf/verifier.c

19862

ret = bpf_check_btf_info_early(env, attr, uattr);

kernel/bpf/verifier.c

19866

ret = add_subprog_and_kfunc(env);

kernel/bpf/verifier.c

19870

ret = check_subprogs(env);

kernel/bpf/verifier.c

19874

ret = bpf_check_btf_info(env, attr, uattr);

kernel/bpf/verifier.c

19878

ret = check_and_resolve_insns(env);

kernel/bpf/verifier.c

19882

if (bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

19883

ret = bpf_prog_offload_verifier_prep(env->prog);

kernel/bpf/verifier.c

19888

ret = bpf_check_cfg(env);

kernel/bpf/verifier.c

19892

ret = bpf_compute_postorder(env);

kernel/bpf/verifier.c

19896

ret = bpf_stack_liveness_init(env);

kernel/bpf/verifier.c

19900

ret = check_attach_btf_id(env);

kernel/bpf/verifier.c

19904

ret = bpf_compute_const_regs(env);

kernel/bpf/verifier.c

19908

ret = bpf_prune_dead_branches(env);

kernel/bpf/verifier.c

19912

ret = sort_subprogs_topo(env);

kernel/bpf/verifier.c

19916

ret = bpf_compute_scc(env);

kernel/bpf/verifier.c

19920

ret = bpf_compute_live_registers(env);

kernel/bpf/verifier.c

19924

ret = mark_fastcall_patterns(env);

kernel/bpf/verifier.c

19928

ret = do_check_main(env);

kernel/bpf/verifier.c

19929

ret = ret ?: do_check_subprogs(env);

kernel/bpf/verifier.c

19931

if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/verifier.c

19932

ret = bpf_prog_offload_finalize(env);

kernel/bpf/verifier.c

19935

kvfree(env->explored_states);

kernel/bpf/verifier.c

19941

ret = bpf_remove_fastcall_spills_fills(env);

kernel/bpf/verifier.c

19944

ret = check_max_stack_depth(env);

kernel/bpf/verifier.c

19948

ret = bpf_optimize_bpf_loop(env);

kernel/bpf/verifier.c

19952

bpf_opt_hard_wire_dead_code_branches(env);

kernel/bpf/verifier.c

19954

ret = bpf_opt_remove_dead_code(env);

kernel/bpf/verifier.c

19956

ret = bpf_opt_remove_nops(env);

kernel/bpf/verifier.c

19959

sanitize_dead_code(env);

kernel/bpf/verifier.c

19964

ret = bpf_convert_ctx_accesses(env);

kernel/bpf/verifier.c

19967

ret = bpf_do_misc_fixups(env);

kernel/bpf/verifier.c

19972

if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

19973

ret = bpf_opt_subreg_zext_lo32_rnd_hi32(env, attr);

kernel/bpf/verifier.c

19974

env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret

kernel/bpf/verifier.c

19979

ret = bpf_fixup_call_args(env);

kernel/bpf/verifier.c

19981

env->verification_time = ktime_get_ns() - start_time;

kernel/bpf/verifier.c

19982

print_verification_stats(env);

kernel/bpf/verifier.c

19983

env->prog->aux->verified_insns = env->insn_processed;

kernel/bpf/verifier.c

19986

err = bpf_log_attr_finalize(attr_log, &env->log);

kernel/bpf/verifier.c

19993

if (env->used_map_cnt) {

kernel/bpf/verifier.c

19995

env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0],

kernel/bpf/verifier.c

19996

env->used_map_cnt,

kernel/bpf/verifier.c

19999

if (!env->prog->aux->used_maps) {

kernel/bpf/verifier.c

20004

memcpy(env->prog->aux->used_maps, env->used_maps,

kernel/bpf/verifier.c

20005

sizeof(env->used_maps[0]) * env->used_map_cnt);

kernel/bpf/verifier.c

20006

env->prog->aux->used_map_cnt = env->used_map_cnt;

kernel/bpf/verifier.c

20008

if (env->used_btf_cnt) {

kernel/bpf/verifier.c

20010

env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0],

kernel/bpf/verifier.c

20011

env->used_btf_cnt,

kernel/bpf/verifier.c

20013

if (!env->prog->aux->used_btfs) {

kernel/bpf/verifier.c

20018

memcpy(env->prog->aux->used_btfs, env->used_btfs,

kernel/bpf/verifier.c

20019

sizeof(env->used_btfs[0]) * env->used_btf_cnt);

kernel/bpf/verifier.c

20020

env->prog->aux->used_btf_cnt = env->used_btf_cnt;

kernel/bpf/verifier.c

20022

if (env->used_map_cnt || env->used_btf_cnt) {

kernel/bpf/verifier.c

20026

convert_pseudo_ld_imm64(env);

kernel/bpf/verifier.c

20029

adjust_btf_func(env);

kernel/bpf/verifier.c

20034

if (env->prog->type == BPF_PROG_TYPE_EXT)

kernel/bpf/verifier.c

20035

env->prog->expected_attach_type = 0;

kernel/bpf/verifier.c

20037

env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);

kernel/bpf/verifier.c

20041

release_insn_arrays(env);

kernel/bpf/verifier.c

20042

if (!env->prog->aux->used_maps)

kernel/bpf/verifier.c

20046

release_maps(env);

kernel/bpf/verifier.c

20047

if (!env->prog->aux->used_btfs)

kernel/bpf/verifier.c

20048

release_btfs(env);

kernel/bpf/verifier.c

20050

*prog = env->prog;

kernel/bpf/verifier.c

20052

module_put(env->attach_btf_mod);

kernel/bpf/verifier.c

20056

bpf_clear_insn_aux_data(env, 0, env->prog->len);

kernel/bpf/verifier.c

20058

bpf_stack_liveness_free(env);

kernel/bpf/verifier.c

20059

kvfree(env->cfg.insn_postorder);

kernel/bpf/verifier.c

20060

kvfree(env->scc_info);

kernel/bpf/verifier.c

20061

kvfree(env->succ);

kernel/bpf/verifier.c

20062

kvfree(env->gotox_tmp_buf);

kernel/bpf/verifier.c

20063

vfree(env->insn_aux_data);

kernel/bpf/verifier.c

20064

kvfree(env);

kernel/bpf/verifier.c

203

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id);

kernel/bpf/verifier.c

205

static int release_reference(struct bpf_verifier_env *env, int id);

kernel/bpf/verifier.c

206

static void invalidate_non_owning_refs(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

207

static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

209

static int ref_set_non_owning(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

211

static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg);

kernel/bpf/verifier.c

2113

static int reg_bounds_sanity_check(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

212

static inline bool in_sleepable_context(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

213

static const char *non_sleepable_context_description(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

2135

verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} "

kernel/bpf/verifier.c

2141

if (env->test_reg_invariants)

kernel/bpf/verifier.c

2162

static void __mark_reg_unknown(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2166

reg->precise = !env->bpf_capable;

kernel/bpf/verifier.c

2169

static void mark_reg_unknown(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2172

__mark_reg_unknown(env, regs + regno);

kernel/bpf/verifier.c

2175

static int __mark_reg_s32_range(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2192

return reg_bounds_sanity_check(env, reg, "s32_range");

kernel/bpf/verifier.c

2195

void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2198

__mark_reg_unknown(env, reg);

kernel/bpf/verifier.c

2202

static int mark_btf_ld_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2210

mark_reg_unknown(env, regs, regno);

kernel/bpf/verifier.c

2213

mark_reg_known_zero(env, regs, regno);

kernel/bpf/verifier.c

2218

regs[regno].id = ++env->id_gen;

kernel/bpf/verifier.c

2221

mark_reg_known_zero(env, regs, regno);

kernel/bpf/verifier.c

2226

verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__);

kernel/bpf/verifier.c

2232

static void init_reg_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2239

bpf_mark_reg_not_init(env, &regs[i]);

kernel/bpf/verifier.c

2245

mark_reg_known_zero(env, regs, BPF_REG_FP);

kernel/bpf/verifier.c

2258

static void init_func_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2266

init_reg_state(env, state);

kernel/bpf/verifier.c

2267

mark_verifier_state_scratched(env);

kernel/bpf/verifier.c

2271

static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2284

elem->next = env->head;

kernel/bpf/verifier.c

2285

elem->log_pos = env->log.end_pos;

kernel/bpf/verifier.c

2286

env->head = elem;

kernel/bpf/verifier.c

2287

env->stack_size++;

kernel/bpf/verifier.c

2288

if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {

kernel/bpf/verifier.c

2289

verbose(env,

kernel/bpf/verifier.c

2291

env->stack_size);

kernel/bpf/verifier.c

2304

init_func_state(env, frame,

kernel/bpf/verifier.c

2320

struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

2322

struct bpf_subprog_info *vals = env->subprog_info;

kernel/bpf/verifier.c

2325

if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)

kernel/bpf/verifier.c

2329

r = env->subprog_cnt - 1;

kernel/bpf/verifier.c

2341

int bpf_find_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

2345

p = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

2348

return p - env->subprog_info;

kernel/bpf/verifier.c

2351

static int add_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

2353

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

2357

verbose(env, "call to invalid destination\n");

kernel/bpf/verifier.c

2360

ret = bpf_find_subprog(env, off);

kernel/bpf/verifier.c

2363

if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {

kernel/bpf/verifier.c

2364

verbose(env, "too many subprograms\n");

kernel/bpf/verifier.c

2368

env->subprog_info[env->subprog_cnt++].start = off;

kernel/bpf/verifier.c

2369

sort(env->subprog_info, env->subprog_cnt,

kernel/bpf/verifier.c

2370

sizeof(env->subprog_info[0]), cmp_subprogs, NULL);

kernel/bpf/verifier.c

2371

return env->subprog_cnt - 1;

kernel/bpf/verifier.c

2374

static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

2376

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

2390

verbose(env, "invalid btf id for main subprog in func_info\n");

kernel/bpf/verifier.c

2401

verbose(env, "multiple exception callback tags for main subprog\n");

kernel/bpf/verifier.c

2407

verbose(env, "exception callback '%s' could not be found in BTF\n", name);

kernel/bpf/verifier.c

2413

verbose(env, "exception callback '%s' must have global linkage\n", name);

kernel/bpf/verifier.c

242

static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *ref_obj)

kernel/bpf/verifier.c

2425

verbose(env, "invalid exception callback insn_off in func_info: 0\n");

kernel/bpf/verifier.c

2430

verbose(env, "exception callback type id not found in func_info\n");

kernel/bpf/verifier.c

245

verifier_bug(env, "function expects only one referenced object but got %d\n",

kernel/bpf/verifier.c

2493

static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2503

tab = env->prog->aux->kfunc_btf_tab;

kernel/bpf/verifier.c

2508

verbose(env, "too many different module BTFs\n");

kernel/bpf/verifier.c

2512

if (bpfptr_is_null(env->fd_array)) {

kernel/bpf/verifier.c

2513

verbose(env, "kfunc offset > 0 without fd_array is invalid\n");

kernel/bpf/verifier.c

2517

if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,

kernel/bpf/verifier.c

2524

verbose(env, "invalid module BTF fd specified\n");

kernel/bpf/verifier.c

2529

verbose(env, "BTF fd for kfunc is not a module BTF\n");

kernel/bpf/verifier.c

2569

static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)

kernel/bpf/verifier.c

2576

verbose(env, "negative offset disallowed for kernel module function call\n");

kernel/bpf/verifier.c

2580

return __find_kfunc_desc_btf(env, offset);

kernel/bpf/verifier.c

2614

static int fetch_kfunc_meta(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2625

verbose(env, "invalid kernel function btf_id %d\n", func_id);

kernel/bpf/verifier.c

2629

btf = find_kfunc_desc_btf(env, offset);

kernel/bpf/verifier.c

2631

verbose(env, "failed to find BTF for kernel function\n");

kernel/bpf/verifier.c

2642

kfunc_flags = btf_kfunc_flags(btf, func_id, env->prog);

kernel/bpf/verifier.c

2646

verbose(env, "kernel btf_id %d is not a function\n", func_id);

kernel/bpf/verifier.c

2657

func_proto = find_kfunc_impl_proto(&env->log, btf, func_name);

kernel/bpf/verifier.c

2662

verbose(env, "kernel function btf_id %d does not have a valid func_proto\n",

kernel/bpf/verifier.c

2677

int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)

kernel/bpf/verifier.c

2688

prog_aux = env->prog->aux;

kernel/bpf/verifier.c

2693

verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");

kernel/bpf/verifier.c

2697

if (!env->prog->jit_requested) {

kernel/bpf/verifier.c

2698

verbose(env, "JIT is required for calling kernel function\n");

kernel/bpf/verifier.c

2703

verbose(env, "JIT does not support calling kernel function\n");

kernel/bpf/verifier.c

2707

if (!env->prog->gpl_compatible) {

kernel/bpf/verifier.c

2708

verbose(env, "cannot call kernel function from non-GPL compatible program\n");

kernel/bpf/verifier.c

2734

if (find_kfunc_desc(env->prog, func_id, offset))

kernel/bpf/verifier.c

2738

verbose(env, "too many different kernel function calls\n");

kernel/bpf/verifier.c

2742

err = fetch_kfunc_meta(env, func_id, offset, &kfunc);

kernel/bpf/verifier.c

2748

verbose(env, "cannot find address for kernel function %s\n", kfunc.name);

kernel/bpf/verifier.c

2753

err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);

kernel/bpf/verifier.c

2758

err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model);

kernel/bpf/verifier.c

2777

static int add_subprog_and_kfunc(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

2779

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

2780

int i, ret, insn_cnt = env->prog->len, ex_cb_insn;

kernel/bpf/verifier.c

2781

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

2784

ret = add_subprog(env, 0);

kernel/bpf/verifier.c

2793

if (!env->bpf_capable) {

kernel/bpf/verifier.c

2794

verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");

kernel/bpf/verifier.c

2799

ret = add_subprog(env, i + insn->imm + 1);

kernel/bpf/verifier.c

2801

ret = bpf_add_kfunc_call(env, insn->imm, insn->off);

kernel/bpf/verifier.c

2807

ret = bpf_find_exception_callback_insn_off(env);

kernel/bpf/verifier.c

2816

ret = add_subprog(env, ex_cb_insn);

kernel/bpf/verifier.c

2819

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

2820

if (env->subprog_info[i].start != ex_cb_insn)

kernel/bpf/verifier.c

2822

env->exception_callback_subprog = i;

kernel/bpf/verifier.c

2823

bpf_mark_subprog_exc_cb(env, i);

kernel/bpf/verifier.c

2831

subprog[env->subprog_cnt].start = insn_cnt;

kernel/bpf/verifier.c

2833

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

2834

for (i = 0; i < env->subprog_cnt; i++)

kernel/bpf/verifier.c

2835

verbose(env, "func#%d @%d\n", i, subprog[i].start);

kernel/bpf/verifier.c

2840

static int check_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

2843

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

2844

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

2845

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

2872

verbose(env, "jump out of range from insn %d to %d\n", i, off);

kernel/bpf/verifier.c

2884

verbose(env, "last insn is not an exit or jmp\n");

kernel/bpf/verifier.c

2889

if (cur_subprog < env->subprog_cnt)

kernel/bpf/verifier.c

2903

static int sort_subprogs_topo(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

2905

struct bpf_subprog_info *si = env->subprog_info;

kernel/bpf/verifier.c

2906

int *insn_postorder = env->cfg.insn_postorder;

kernel/bpf/verifier.c

2907

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

2908

int cnt = env->subprog_cnt;

kernel/bpf/verifier.c

2944

callee = bpf_find_subprog(env, idx + insn[idx].imm + 1);

kernel/bpf/verifier.c

2954

verbose(env, "recursive call from %s() to %s()\n",

kernel/bpf/verifier.c

2955

subprog_name(env, cur),

kernel/bpf/verifier.c

2956

subprog_name(env, callee));

kernel/bpf/verifier.c

2968

env->subprog_topo_order[order++] = cur;

kernel/bpf/verifier.c

2974

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

2976

verbose(env, "topo_order[%d] = %s\n",

kernel/bpf/verifier.c

2977

i, subprog_name(env, env->subprog_topo_order[i]));

kernel/bpf/verifier.c

2984

static void mark_stack_slots_scratched(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2990

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

3083

static void mark_insn_zext(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3091

env->insn_aux_data[def_idx - 1].zext_dst = true;

kernel/bpf/verifier.c

3096

static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,

kernel/bpf/verifier.c

3099

struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;

kernel/bpf/verifier.c

3103

mark_reg_scratched(env, regno);

kernel/bpf/verifier.c

3110

verbose(env, "R%d !read_ok\n", regno);

kernel/bpf/verifier.c

3118

mark_insn_zext(env, reg);

kernel/bpf/verifier.c

3124

verbose(env, "frame pointer is read only\n");

kernel/bpf/verifier.c

3127

reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;

kernel/bpf/verifier.c

3129

mark_reg_unknown(env, regs, regno);

kernel/bpf/verifier.c

3134

static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

3137

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

3140

return __check_reg_arg(env, state->regs, regno, t);

kernel/bpf/verifier.c

3143

static void mark_indirect_target(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

3145

env->insn_aux_data[idx].indirect_target = true;

kernel/bpf/verifier.c

3247

void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

3252

.private_data = env,

kernel/bpf/verifier.c

3255

print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);

kernel/bpf/verifier.c

329

struct bpf_verifier_env *env = private_data;

kernel/bpf/verifier.c

3294

int mark_chain_precision(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

3296

return bpf_mark_chain_precision(env, env->cur_state, regno, NULL);

kernel/bpf/verifier.c

3302

static int mark_chain_precision_batch(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3305

return bpf_mark_chain_precision(env, starting_state, -1, NULL);

kernel/bpf/verifier.c

332

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/verifier.c

336

bpf_verifier_vlog(&env->log, fmt, args);

kernel/bpf/verifier.c

3364

static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3382

src_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

3385

static void save_register_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3399

mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);

kernel/bpf/verifier.c

340

static void verbose_invalid_scalar(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3413

static void check_fastcall_stack_contract(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3416

struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];

kernel/bpf/verifier.c

3417

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

3452

static int check_stack_write_fixed_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3460

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

3468

if (!env->allow_ptr_leaks &&

kernel/bpf/verifier.c

347

verbose(env, "%s the register %s has", ctx, reg_name);

kernel/bpf/verifier.c

3472

verbose(env, "attempt to corrupt spilled pointer on stack\n");

kernel/bpf/verifier.c

3476

cur = env->cur_state->frame[env->cur_state->curframe];

kernel/bpf/verifier.c

3479

if (!env->bypass_spec_v4) {

kernel/bpf/verifier.c

349

verbose(env, " smin=%lld", reg_smin(reg));

kernel/bpf/verifier.c

3493

env->insn_aux_data[insn_idx].nospec_result = true;

kernel/bpf/verifier.c

3496

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

3500

check_fastcall_stack_contract(env, state, insn_idx, off);

kernel/bpf/verifier.c

3501

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

3502

if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {

kernel/bpf/verifier.c

3508

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

3509

save_register_state(env, state, spi, reg, size);

kernel/bpf/verifier.c

3514

env->bpf_capable) {

kernel/bpf/verifier.c

3515

struct bpf_reg_state *tmp_reg = &env->fake_reg[0];

kernel/bpf/verifier.c

3520

save_register_state(env, state, spi, tmp_reg, size);

kernel/bpf/verifier.c

3524

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/verifier.c

3525

verbose(env, "invalid size of register spill\n");

kernel/bpf/verifier.c

3529

verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");

kernel/bpf/verifier.c

353

verbose(env, " smax=%lld", reg_smax(reg));

kernel/bpf/verifier.c

3532

save_register_state(env, state, spi, reg, size);

kernel/bpf/verifier.c

3547

err = mark_chain_precision(env, value_regno);

kernel/bpf/verifier.c

3560

return bpf_push_jmp_history(env, env->cur_state, insn_flags,

kernel/bpf/verifier.c

357

verbose(env, " unknown scalar value");

kernel/bpf/verifier.c

358

verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);

kernel/bpf/verifier.c

3583

static int check_stack_write_var_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3593

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

3600

cur = env->cur_state->frame[env->cur_state->curframe];

kernel/bpf/verifier.c

361

static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)

kernel/bpf/verifier.c

3613

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

3618

check_fastcall_stack_contract(env, state, insn_idx, min_off);

kernel/bpf/verifier.c

3627

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

3629

if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {

kernel/bpf/verifier.c

3641

verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",

kernel/bpf/verifier.c

3683

!env->allow_uninit_stack) {

kernel/bpf/verifier.c

3684

verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",

kernel/bpf/verifier.c

3692

err = mark_chain_precision(env, value_regno);

kernel/bpf/verifier.c

3707

static void mark_reg_stack_read(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3712

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

3721

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

3731

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

3734

mark_reg_unknown(env, state->regs, dst_regno);

kernel/bpf/verifier.c

3747

static int check_stack_read_fixed_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

375

(type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) ||

kernel/bpf/verifier.c

3752

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

3763

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

3764

check_fastcall_stack_contract(env, state, env->insn_idx, off);

kernel/bpf/verifier.c

3774

verbose_linfo(env, env->insn_idx, "; ");

kernel/bpf/verifier.c

3775

verbose(env, "invalid size of register fill\n");

kernel/bpf/verifier.c

3789

if (env->bpf_capable && size == 4 && spill_size == 4 &&

kernel/bpf/verifier.c

3794

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

3818

if (type == STACK_INVALID && env->allow_uninit_stack)

kernel/bpf/verifier.c

3821

verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n",

kernel/bpf/verifier.c

3824

verbose(env, "invalid read from stack off %d+%d size %d\n",

kernel/bpf/verifier.c

3832

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

3836

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

3839

mark_reg_unknown(env, state->regs, dst_regno);

kernel/bpf/verifier.c

3845

if (env->bpf_capable)

kernel/bpf/verifier.c

3849

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

3855

} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {

kernel/bpf/verifier.c

3862

verbose(env, "leaking pointer from stack off %d\n",

kernel/bpf/verifier.c

3873

if (type == STACK_INVALID && env->allow_uninit_stack)

kernel/bpf/verifier.c

3876

verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n",

kernel/bpf/verifier.c

3879

verbose(env, "invalid read from stack off %d+%d size %d\n",

kernel/bpf/verifier.c

3885

mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);

kernel/bpf/verifier.c

3889

return bpf_push_jmp_history(env, env->cur_state, insn_flags,

kernel/bpf/verifier.c

3899

static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

3905

static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

3907

return cur_regs(env) + regno;

kernel/bpf/verifier.c

3923

static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

3926

struct bpf_func_state *ptr_state = bpf_func(env, reg);

kernel/bpf/verifier.c

3932

err = check_stack_range_initialized(env, reg, ptr_argno, off, size,

kernel/bpf/verifier.c

3939

mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);

kernel/bpf/verifier.c

3940

check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);

kernel/bpf/verifier.c

395

bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

3953

static int check_stack_read(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3957

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

397

struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;

kernel/bpf/verifier.c

3970

verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",

kernel/bpf/verifier.c

3985

err = check_stack_read_fixed_off(env, state, off, size,

kernel/bpf/verifier.c

3992

err = check_stack_read_var_off(env, reg, ptr_argno, off, size,

kernel/bpf/verifier.c

4008

static int check_stack_write(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4012

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

4017

err = check_stack_write_fixed_off(env, state, off, size,

kernel/bpf/verifier.c

402

static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4023

err = check_stack_write_var_off(env, state,

kernel/bpf/verifier.c

4034

static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state,

kernel/bpf/verifier.c

4038

struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];

kernel/bpf/verifier.c

4044

verbose(env, "stack arg write offset %d exceeds max %d stack args\n",

kernel/bpf/verifier.c

4049

err = grow_stack_arg_slots(env, state, spi + 1);

kernel/bpf/verifier.c

405

const struct btf *btf = env->prog->aux->btf;

kernel/bpf/verifier.c

4063

__mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);

kernel/bpf/verifier.c

4066

return bpf_push_jmp_history(env, env->cur_state,

kernel/bpf/verifier.c

4074

static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state,

kernel/bpf/verifier.c

4077

struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];

kernel/bpf/verifier.c

4078

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

408

btf_id = env->prog->aux->func_info[subprog].type_id;

kernel/bpf/verifier.c

4084

verbose(env, "r11 load must be before any r11 store or call insn\n");

kernel/bpf/verifier.c

4089

verbose(env, "invalid read from stack arg off %d depth %d\n",

kernel/bpf/verifier.c

4098

return bpf_push_jmp_history(env, env->cur_state,

kernel/bpf/verifier.c

4102

static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)

kernel/bpf/verifier.c

4104

struct bpf_func_state *caller = cur_func(env);

kernel/bpf/verifier.c

4107

bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi);

kernel/bpf/verifier.c

4108

return mark_chain_precision_batch(env, env->cur_state);

kernel/bpf/verifier.c

411

if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id))

kernel/bpf/verifier.c

4111

static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,

kernel/bpf/verifier.c

4120

verbose(env, "callee expects %d args, stack arg%d is not initialized\n",

kernel/bpf/verifier.c

4138

static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

4145

verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n",

kernel/bpf/verifier.c

4151

verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n",

kernel/bpf/verifier.c

4160

static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

4171

verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

4175

verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

4181

verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n",

kernel/bpf/verifier.c

4182

off, size, reg_arg_name(env, argno), reg->id, off, mem_size);

kernel/bpf/verifier.c

4185

verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

4190

verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",

kernel/bpf/verifier.c

4198

static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

4218

verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",

kernel/bpf/verifier.c

4219

reg_arg_name(env, argno));

kernel/bpf/verifier.c

4222

err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size,

kernel/bpf/verifier.c

4225

verbose(env, "%s min value is outside of the allowed memory range\n",

kernel/bpf/verifier.c

4226

reg_arg_name(env, argno));

kernel/bpf/verifier.c

4235

verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n",

kernel/bpf/verifier.c

4236

reg_arg_name(env, argno));

kernel/bpf/verifier.c

4239

err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size,

kernel/bpf/verifier.c

4242

verbose(env, "%s max value is outside of the allowed memory range\n",

kernel/bpf/verifier.c

4243

reg_arg_name(env, argno));

kernel/bpf/verifier.c

425

static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4250

static int __check_ptr_off_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4262

verbose(env, "variable %s access var_off=%s disallowed\n",

kernel/bpf/verifier.c

4263

reg_type_str(env, reg->type), tn_buf);

kernel/bpf/verifier.c

4268

verbose(env, "negative offset %s ptr %s off=%lld disallowed\n",

kernel/bpf/verifier.c

4269

reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);

kernel/bpf/verifier.c

4274

verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n",

kernel/bpf/verifier.c

4275

reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);

kernel/bpf/verifier.c

4282

static int check_ptr_off_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4285

return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);

kernel/bpf/verifier.c

4288

static int map_kptr_match_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

429

if (!env->prog->aux->func_info)

kernel/bpf/verifier.c

432

info = &env->prog->aux->func_info[subprog];

kernel/bpf/verifier.c

4323

if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true))

kernel/bpf/verifier.c

433

return btf_type_name(env->prog->aux->btf, info->type_id);

kernel/bpf/verifier.c

4350

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value,

kernel/bpf/verifier.c

4356

verbose(env, "invalid kptr access, R%d type=%s%s ", regno,

kernel/bpf/verifier.c

4357

reg_type_str(env, reg->type), reg_name);

kernel/bpf/verifier.c

4358

verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);

kernel/bpf/verifier.c

436

void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4360

verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),

kernel/bpf/verifier.c

4363

verbose(env, "\n");

kernel/bpf/verifier.c

4367

static bool in_sleepable(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

4369

return env->cur_state->in_sleepable;

kernel/bpf/verifier.c

4375

static bool in_rcu_cs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

4377

return env->cur_state->active_rcu_locks ||

kernel/bpf/verifier.c

4378

env->cur_state->active_locks ||

kernel/bpf/verifier.c

4379

!in_sleepable(env);

kernel/bpf/verifier.c

438

struct bpf_subprog_info *info = subprog_info(env, subprog);

kernel/bpf/verifier.c

4427

static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)

kernel/bpf/verifier.c

4433

if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {

kernel/bpf/verifier.c

445

static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4450

static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

4457

mark_reg_known_zero(env, cur_regs(env), regno);

kernel/bpf/verifier.c

4458

reg = reg_state(env, regno);

kernel/bpf/verifier.c

4461

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

4466

static int check_map_kptr_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

447

return subprog_info(env, subprog)->is_exception_cb;

kernel/bpf/verifier.c

4470

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

4483

verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");

kernel/bpf/verifier.c

4492

verbose(env, "store to referenced kptr disallowed\n");

kernel/bpf/verifier.c

4496

verbose(env, "store to uptr disallowed\n");

kernel/bpf/verifier.c

4502

return mark_uptr_ld_reg(env, value_regno, kptr_field);

kernel/bpf/verifier.c

4507

ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID,

kernel/bpf/verifier.c

4509

btf_ld_kptr_type(env, kptr_field));

kernel/bpf/verifier.c

4513

val_reg = reg_state(env, value_regno);

kernel/bpf/verifier.c

4515

map_kptr_match_type(env, kptr_field, val_reg, value_regno))

kernel/bpf/verifier.c

4519

verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",

kernel/bpf/verifier.c

4524

verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");

kernel/bpf/verifier.c

4543

static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

4552

err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed);

kernel/bpf/verifier.c

4575

verbose(env, "%s cannot be accessed indirectly by helper\n",

kernel/bpf/verifier.c

4580

verbose(env, "%s access cannot have variable offset\n",

kernel/bpf/verifier.c

4585

verbose(env, "%s access misaligned expected=%u off=%llu\n",

kernel/bpf/verifier.c

4591

verbose(env, "%s access size must be BPF_DW\n",

kernel/bpf/verifier.c

4597

verbose(env, "%s cannot be accessed directly by load/store\n",

kernel/bpf/verifier.c

4606

static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4610

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

4634

env->seen_direct_write = true;

kernel/bpf/verifier.c

4639

env->seen_direct_write = true;

kernel/bpf/verifier.c

4648

static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,

kernel/bpf/verifier.c

4654

verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

4658

err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed);

kernel/bpf/verifier.c

4668

env->prog->aux->max_pkt_offset =

kernel/bpf/verifier.c

4669

max_t(u32, env->prog->aux->max_pkt_offset,

kernel/bpf/verifier.c

4681

static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,

kernel/bpf/verifier.c

4684

if (env->ops->is_valid_access &&

kernel/bpf/verifier.c

4685

env->ops->is_valid_access(off, size, t, env->prog, info)) {

kernel/bpf/verifier.c

4695

!find_reference_state(env->cur_state, info->ref_id)) {

kernel/bpf/verifier.c

4696

verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",

kernel/bpf/verifier.c

4701

env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;

kernel/bpf/verifier.c

4704

if (env->prog->aux->max_ctx_offset < off + size)

kernel/bpf/verifier.c

4705

env->prog->aux->max_ctx_offset = off + size;

kernel/bpf/verifier.c

4709

verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);

kernel/bpf/verifier.c

4713

static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

4721

bool var_off_ok = is_var_ctx_off_allowed(env->prog);

kernel/bpf/verifier.c

4722

bool fixed_off_ok = !env->ops->convert_ctx_access;

kernel/bpf/verifier.c

4726

err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false);

kernel/bpf/verifier.c

4728

err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok);

kernel/bpf/verifier.c

4733

err = __check_ctx_access(env, insn_idx, off, access_size, t, info);

kernel/bpf/verifier.c

4735

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/verifier.c

4739

static int check_flow_keys_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4748

verbose(env, "%s invalid variable offset to flow keys: off=%d, var_off=%s\n",

kernel/bpf/verifier.c

4749

reg_arg_name(env, argno), off, tn_buf);

kernel/bpf/verifier.c

4756

verbose(env, "invalid access to flow keys off=%d size=%d\n",

kernel/bpf/verifier.c

4763

static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,

kernel/bpf/verifier.c

4771

verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",

kernel/bpf/verifier.c

4772

reg_arg_name(env, argno));

kernel/bpf/verifier.c

4795

env->insn_aux_data[insn_idx].ctx_field_size =

kernel/bpf/verifier.c

4800

verbose(env, "%s invalid %s access off=%d size=%d\n",

kernel/bpf/verifier.c

4801

reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size);

kernel/bpf/verifier.c

4806

static bool is_pointer_value(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4808

return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));

kernel/bpf/verifier.c

4811

static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4813

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

4818

static bool is_sk_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4820

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

4825

static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4827

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

4832

static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4834

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

4840

static bool is_arena_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

4842

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

4850

static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

4853

if (is_ctx_reg(env, regno))

kernel/bpf/verifier.c

4855

if (is_pkt_reg(env, regno))

kernel/bpf/verifier.c

4857

if (is_flow_key_reg(env, regno))

kernel/bpf/verifier.c

4859

if (is_sk_reg(env, regno))

kernel/bpf/verifier.c

4861

if (is_arena_reg(env, regno))

kernel/bpf/verifier.c

4876

static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)

kernel/bpf/verifier.c

4879

if (reg_is_referenced(env, reg))

kernel/bpf/verifier.c

4910

static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4936

verbose(env,

kernel/bpf/verifier.c

4945

static int check_generic_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4961

verbose(env, "misaligned %saccess off %s+%d size %d\n",

kernel/bpf/verifier.c

4969

static int check_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4973

bool strict = env->strict_alignment || strict_alignment_once;

kernel/bpf/verifier.c

4982

return check_pkt_ptr_alignment(env, reg, off, size, strict);

kernel/bpf/verifier.c

5022

return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,

kernel/bpf/verifier.c

5054

static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)

kernel/bpf/verifier.c

5056

if (env->prog->jit_requested)

kernel/bpf/verifier.c

5076

static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,

kernel/bpf/verifier.c

5080

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

5081

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

5114

verbose(env,

kernel/bpf/verifier.c

5120

subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

5135

if (subprog_depth > env->max_stack_depth)

kernel/bpf/verifier.c

5136

env->max_stack_depth = subprog_depth;

kernel/bpf/verifier.c

5138

verbose(env, "stack size of subprog %d is %d. Too large\n",

kernel/bpf/verifier.c

5144

if (depth > env->max_stack_depth)

kernel/bpf/verifier.c

5145

env->max_stack_depth = depth;

kernel/bpf/verifier.c

5151

verbose(env, "combined stack size of %d calls is %d. Too large\n",

kernel/bpf/verifier.c

5174

verbose(env,

kernel/bpf/verifier.c

5186

sidx = bpf_find_subprog(env, next_insn);

kernel/bpf/verifier.c

5187

if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))

kernel/bpf/verifier.c

5191

verifier_bug(env, "subprog has tail_call and async cb");

kernel/bpf/verifier.c

5198

verbose(env, "insn %d cannot call exception cb directly", i);

kernel/bpf/verifier.c

5219

frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1;

kernel/bpf/verifier.c

5221

verbose(env, "the call stack of %d frames is too deep !\n",

kernel/bpf/verifier.c

5235

verbose(env, "cannot tail call within exception cb\n");

kernel/bpf/verifier.c

5239

verbose(env, "tail_calls are not allowed in programs with stack args\n");

kernel/bpf/verifier.c

5245

verbose(env, "tail_calls are not allowed in programs with stack args\n");

kernel/bpf/verifier.c

5250

env->prog->aux->tail_call_reachable = true;

kernel/bpf/verifier.c

5258

depth -= round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

5273

static int check_max_stack_depth(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

5277

struct bpf_subprog_info *si = env->subprog_info;

kernel/bpf/verifier.c

5281

dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

5285

for (int i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

5293

priv_stack_mode = bpf_enable_priv_stack(env->prog);

kernel/bpf/verifier.c

530

static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

5303

for (int i = env->subprog_cnt - 1; i >= 0; i--) {

kernel/bpf/verifier.c

5306

ret = check_max_stack_depth_subprog(env, i, dinfo,

kernel/bpf/verifier.c

5315

for (int i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

5317

env->prog->aux->jits_use_priv_stack = true;

kernel/bpf/verifier.c

5327

static int __check_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5339

verbose(env,

kernel/bpf/verifier.c

5341

reg_arg_name(env, argno), off, tn_buf);

kernel/bpf/verifier.c

5347

verbose(env,

kernel/bpf/verifier.c

5349

reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value);

kernel/bpf/verifier.c

5357

static int check_tp_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5364

err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end);

kernel/bpf/verifier.c

5368

env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);

kernel/bpf/verifier.c

5373

static int check_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5383

err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end);

kernel/bpf/verifier.c

541

verifier_bug(env, "unhandled async callback in is_async_cb_sleepable");

kernel/bpf/verifier.c

564

static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

5686

static bool type_is_rcu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5695

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");

kernel/bpf/verifier.c

5698

static bool type_is_rcu_or_null(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

570

verbose(env, "%s has to be at a constant offset\n", obj_kind);

kernel/bpf/verifier.c

5706

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");

kernel/bpf/verifier.c

5709

static bool type_is_trusted(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5718

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");

kernel/bpf/verifier.c

5721

static bool type_is_trusted_or_null(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5729

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,

kernel/bpf/verifier.c

5733

static int check_ptr_to_btf_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5746

if (!env->allow_ptr_leaks) {

kernel/bpf/verifier.c

5747

verbose(env,

kernel/bpf/verifier.c

5752

if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {

kernel/bpf/verifier.c

5753

verbose(env,

kernel/bpf/verifier.c

576

verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);

kernel/bpf/verifier.c

5763

verbose(env,

kernel/bpf/verifier.c

5765

reg_arg_name(env, argno), tname, off, tn_buf);

kernel/bpf/verifier.c

5772

verbose(env,

kernel/bpf/verifier.c

5774

reg_arg_name(env, argno), tname, off);

kernel/bpf/verifier.c

5779

verbose(env,

kernel/bpf/verifier.c

5781

reg_arg_name(env, argno), tname, off);

kernel/bpf/verifier.c

5786

verbose(env,

kernel/bpf/verifier.c

5788

reg_arg_name(env, argno), tname, off);

kernel/bpf/verifier.c

5792

if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {

kernel/bpf/verifier.c

5794

verifier_bug(env, "reg->btf must be kernel btf");

kernel/bpf/verifier.c

5797

ret = env->ops->btf_struct_access(&env->log, reg, off, size);

kernel/bpf/verifier.c

5799

verbose(env,

kernel/bpf/verifier.c

5801

reg_arg_name(env, argno), tname, off, size);

kernel/bpf/verifier.c

5808

verbose(env, "only read is supported\n");

kernel/bpf/verifier.c

5813

!(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) {

kernel/bpf/verifier.c

5814

verifier_bug(env, "allocated object must have a referenced id");

kernel/bpf/verifier.c

5818

ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);

kernel/bpf/verifier.c

582

verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);

kernel/bpf/verifier.c

5833

} else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) {

kernel/bpf/verifier.c

5846

if (type_is_trusted(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

5848

} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

5850

} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {

kernel/bpf/verifier.c

5851

if (type_is_rcu(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

5855

type_is_rcu_or_null(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

586

if (!is_spi_bounds_valid(bpf_func(env, reg), spi, nr_slots))

kernel/bpf/verifier.c

5860

if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&

kernel/bpf/verifier.c

5885

ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);

kernel/bpf/verifier.c

5893

static int check_ptr_to_map_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5908

verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");

kernel/bpf/verifier.c

591

static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

5913

verbose(env, "map_ptr access not supported for map type %d\n",

kernel/bpf/verifier.c

5921

if (!env->allow_ptr_leaks) {

kernel/bpf/verifier.c

5922

verbose(env,

kernel/bpf/verifier.c

5929

verbose(env, "%s is %s invalid negative access: off=%d\n",

kernel/bpf/verifier.c

593

return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);

kernel/bpf/verifier.c

5930

reg_arg_name(env, argno), tname, off);

kernel/bpf/verifier.c

5935

verbose(env, "only read from %s is supported\n", tname);

kernel/bpf/verifier.c

5941

ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID,

kernel/bpf/verifier.c

5945

ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);

kernel/bpf/verifier.c

5950

ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);

kernel/bpf/verifier.c

596

static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)

kernel/bpf/verifier.c

5964

static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5971

if (t == BPF_WRITE || env->allow_uninit_stack)

kernel/bpf/verifier.c

598

return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);

kernel/bpf/verifier.c

5987

struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

5991

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

6007

verbose(env, "invalid unbounded variable-offset%s stack %s\n",

kernel/bpf/verifier.c

6008

err_extra, reg_arg_name(env, argno));

kernel/bpf/verifier.c

601

static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

6015

err = check_stack_slot_within_bounds(env, min_off, state, type);

kernel/bpf/verifier.c

6026

verbose(env, "invalid%s stack %s off=%lld size=%d\n",

kernel/bpf/verifier.c

6027

err_extra, reg_arg_name(env, argno), min_off, access_size);

kernel/bpf/verifier.c

603

return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);

kernel/bpf/verifier.c

6032

verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n",

kernel/bpf/verifier.c

6033

err_extra, reg_arg_name(env, argno), tn_buf, off, access_size);

kernel/bpf/verifier.c

6041

return grow_stack_state(env, state, -min_off /* size */);

kernel/bpf/verifier.c

6078

static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

6082

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

6089

err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);

kernel/bpf/verifier.c

6095

verbose(env, "write to change key %s not allowed\n",

kernel/bpf/verifier.c

6096

reg_arg_name(env, argno));

kernel/bpf/verifier.c

6100

err = check_mem_region_access(env, reg, argno, off, size,

kernel/bpf/verifier.c

6105

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6110

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

6111

verbose(env, "R%d leaks addr into map\n", value_regno);

kernel/bpf/verifier.c

6114

err = check_map_access_type(env, reg, off, size, t);

kernel/bpf/verifier.c

6117

err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT);

kernel/bpf/verifier.c

6124

err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field);

kernel/bpf/verifier.c

6148

verbose(env, "Invalid read of %d bytes from insn_array\n",

kernel/bpf/verifier.c

6156

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6164

verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),

kernel/bpf/verifier.c

6165

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6170

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6171

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6176

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

6177

verbose(env, "R%d leaks addr into mem\n", value_regno);

kernel/bpf/verifier.c

6186

err = check_mem_region_access(env, reg, argno, off, size,

kernel/bpf/verifier.c

6189

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6194

.log = &env->log,

kernel/bpf/verifier.c

6199

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

6200

verbose(env, "R%d leaks addr into ctx\n", value_regno);

kernel/bpf/verifier.c

6204

err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info);

kernel/bpf/verifier.c

6211

if (info.is_retval && get_func_retval_range(env->prog, &range)) {

kernel/bpf/verifier.c

6212

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6213

err = __mark_reg_s32_range(env, regs, value_regno,

kernel/bpf/verifier.c

6218

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6221

mark_reg_known_zero(env, regs,

kernel/bpf/verifier.c

6235

regs[value_regno].id = ++env->id_gen;

kernel/bpf/verifier.c

6242

err = check_stack_access_within_bounds(env, reg, argno, off, size, t);

kernel/bpf/verifier.c

6247

err = check_stack_read(env, reg, argno, off, size,

kernel/bpf/verifier.c

6250

err = check_stack_write(env, reg, off, size,

kernel/bpf/verifier.c

6253

if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {

kernel/bpf/verifier.c

6254

verbose(env, "cannot write into packet\n");

kernel/bpf/verifier.c

6258

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

6259

verbose(env, "R%d leaks addr into packet\n",

kernel/bpf/verifier.c

6263

err = check_packet_access(env, reg, argno, off, size, false);

kernel/bpf/verifier.c

6265

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6268

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

6269

verbose(env, "R%d leaks addr into flow keys\n",

kernel/bpf/verifier.c

6274

err = check_flow_keys_access(env, reg, argno, off, size);

kernel/bpf/verifier.c

6276

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6279

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6280

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6283

err = check_sock_access(env, insn_idx, reg, argno, off, size, t);

kernel/bpf/verifier.c

6285

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6287

err = check_tp_buffer_access(env, reg, argno, off, size);

kernel/bpf/verifier.c

6289

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6292

err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t,

kernel/bpf/verifier.c

6295

err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t,

kernel/bpf/verifier.c

6304

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6305

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6308

max_access = &env->prog->aux->max_rdonly_access;

kernel/bpf/verifier.c

6310

max_access = &env->prog->aux->max_rdwr_access;

kernel/bpf/verifier.c

6313

err = check_buffer_access(env, reg, argno, off, size, false,

kernel/bpf/verifier.c

6317

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6320

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

6322

verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),

kernel/bpf/verifier.c

6323

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6338

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,

kernel/bpf/verifier.c

6341

static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

6345

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

6347

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

6353

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

6356

return check_stack_arg_read(env, state, insn->off, insn->dst_reg);

kernel/bpf/verifier.c

6360

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

6365

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

6374

err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off,

kernel/bpf/verifier.c

6377

err = err ?: save_aux_ptr_type(env, src_reg_type,

kernel/bpf/verifier.c

6379

err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);

kernel/bpf/verifier.c

6384

static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

6387

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

6389

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

6395

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

6398

return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg);

kernel/bpf/verifier.c

6402

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

6407

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

6414

err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off,

kernel/bpf/verifier.c

6417

err = err ?: save_aux_ptr_type(env, dst_reg_type, false);

kernel/bpf/verifier.c

6422

static int check_atomic_rmw(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6430

verbose(env, "invalid atomic operand size\n");

kernel/bpf/verifier.c

6435

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

6440

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

6448

err = check_reg_arg(env, aux_reg, SRC_OP);

kernel/bpf/verifier.c

6452

if (is_pointer_value(env, aux_reg)) {

kernel/bpf/verifier.c

6453

verbose(env, "R%d leaks addr into mem\n", aux_reg);

kernel/bpf/verifier.c

6458

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

6459

verbose(env, "R%d leaks addr into mem\n", insn->src_reg);

kernel/bpf/verifier.c

6463

if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {

kernel/bpf/verifier.c

6464

verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",

kernel/bpf/verifier.c

6466

reg_type_str(env, reg_state(env, insn->dst_reg)->type));

kernel/bpf/verifier.c

6477

err = check_reg_arg(env, load_reg, DST_OP);

kernel/bpf/verifier.c

6487

dst_reg = cur_regs(env) + insn->dst_reg;

kernel/bpf/verifier.c

6492

err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,

kernel/bpf/verifier.c

6495

err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg),

kernel/bpf/verifier.c

6501

if (is_arena_reg(env, insn->dst_reg)) {

kernel/bpf/verifier.c

6502

err = save_aux_ptr_type(env, PTR_TO_ARENA, false);

kernel/bpf/verifier.c

6507

err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,

kernel/bpf/verifier.c

6514

static int check_atomic_load(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6519

err = check_load_mem(env, insn, true, false, false, "atomic_load");

kernel/bpf/verifier.c

6523

if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {

kernel/bpf/verifier.c

6524

verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",

kernel/bpf/verifier.c

6526

reg_type_str(env, reg_state(env, insn->src_reg)->type));

kernel/bpf/verifier.c

6533

static int check_atomic_store(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6538

err = check_store_reg(env, insn, true);

kernel/bpf/verifier.c

6542

if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {

kernel/bpf/verifier.c

6543

verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",

kernel/bpf/verifier.c

6545

reg_type_str(env, reg_state(env, insn->dst_reg)->type));

kernel/bpf/verifier.c

6552

static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

656

static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6565

return check_atomic_rmw(env, insn);

kernel/bpf/verifier.c

6568

verbose(env,

kernel/bpf/verifier.c

6572

return check_atomic_load(env, insn);

kernel/bpf/verifier.c

6575

verbose(env,

kernel/bpf/verifier.c

6579

return check_atomic_store(env, insn);

kernel/bpf/verifier.c

6581

verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",

kernel/bpf/verifier.c

6596

struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,

kernel/bpf/verifier.c

6600

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

661

int id = ++env->id_gen;

kernel/bpf/verifier.c

6617

verbose(env, "invalid zero-sized read\n");

kernel/bpf/verifier.c

6621

err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type);

kernel/bpf/verifier.c

6634

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

6638

verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n",

kernel/bpf/verifier.c

6639

reg_arg_name(env, argno), tn_buf);

kernel/bpf/verifier.c

667

static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6677

verbose(env, "potential write to dynptr at off=%d disallowed\n", i);

kernel/bpf/verifier.c

6692

verbose(env, "allocated_stack too small\n");

kernel/bpf/verifier.c

6700

(*stype == STACK_INVALID && env->allow_uninit_stack)) {

kernel/bpf/verifier.c

671

__mark_dynptr_reg(reg, type, true, ++env->id_gen, 0);

kernel/bpf/verifier.c

6710

env->allow_ptr_leaks)) {

kernel/bpf/verifier.c

6712

__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);

kernel/bpf/verifier.c

6722

verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n",

kernel/bpf/verifier.c

6723

reg_arg_name(env, argno), min_off, i - min_off, access_size);

kernel/bpf/verifier.c

6725

verbose(env, "invalid read from stack %s off %d+%d size %d\n",

kernel/bpf/verifier.c

6726

reg_arg_name(env, argno), min_off, i - min_off, access_size);

kernel/bpf/verifier.c

6731

verbose(env, "invalid read from stack %s var_off %s+%d size %d\n",

kernel/bpf/verifier.c

6732

reg_arg_name(env, argno), tn_buf, i - min_off, access_size);

kernel/bpf/verifier.c

674

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6741

static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

6746

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

6752

return check_packet_access(env, reg, argno, 0, access_size,

kernel/bpf/verifier.c

6756

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6757

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6760

return check_mem_region_access(env, reg, argno, 0, access_size,

kernel/bpf/verifier.c

6763

if (check_map_access_type(env, reg, 0, access_size, access_type))

kernel/bpf/verifier.c

6765

return check_map_access(env, reg, argno, 0, access_size,

kernel/bpf/verifier.c

677

static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

6770

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6771

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6775

return check_mem_region_access(env, reg, argno, 0,

kernel/bpf/verifier.c

6781

verbose(env, "%s cannot write into %s\n",

kernel/bpf/verifier.c

6782

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6786

max_access = &env->prog->aux->max_rdonly_access;

kernel/bpf/verifier.c

6788

max_access = &env->prog->aux->max_rdwr_access;

kernel/bpf/verifier.c

6790

return check_buffer_access(env, reg, argno, 0,

kernel/bpf/verifier.c

6795

env, reg,

kernel/bpf/verifier.c

6799

return check_ptr_to_btf_access(env, regs, reg, argno, 0,

kernel/bpf/verifier.c

6803

if (is_var_ctx_off_allowed(env->prog)) {

kernel/bpf/verifier.c

6804

int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX,

kernel/bpf/verifier.c

6808

if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size)

kernel/bpf/verifier.c

6809

env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size;

kernel/bpf/verifier.c

681

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

6819

verbose(env, "%s type=%s ", reg_arg_name(env, argno),

kernel/bpf/verifier.c

6820

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

6821

verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));

kernel/bpf/verifier.c

6831

static int check_mem_size_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

685

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

6859

verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n",

kernel/bpf/verifier.c

6860

reg_arg_name(env, size_argno));

kernel/bpf/verifier.c

6865

verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n",

kernel/bpf/verifier.c

6866

reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg));

kernel/bpf/verifier.c

6871

verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n",

kernel/bpf/verifier.c

6872

reg_arg_name(env, size_argno));

kernel/bpf/verifier.c

6875

err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),

kernel/bpf/verifier.c

6881

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

6883

err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));

kernel/bpf/verifier.c

6888

static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

6899

verbose(env, "%s memory size %u is too large\n",

kernel/bpf/verifier.c

6900

reg_arg_name(env, argno), mem_size);

kernel/bpf/verifier.c

6915

err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL);

kernel/bpf/verifier.c

6916

err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL);

kernel/bpf/verifier.c

6924

static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg,

kernel/bpf/verifier.c

6939

err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta);

kernel/bpf/verifier.c

6940

err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta);

kernel/bpf/verifier.c

6976

static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags)

kernel/bpf/verifier.c

698

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

6980

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

6991

verbose(env,

kernel/bpf/verifier.c

6993

reg_arg_name(env, argno), lock_str);

kernel/bpf/verifier.c

6999

verbose(env,

kernel/bpf/verifier.c

701

err = destroy_if_dynptr_stack_slot(env, state, spi - 1);

kernel/bpf/verifier.c

7010

verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local",

kernel/bpf/verifier.c

7016

verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n",

kernel/bpf/verifier.c

7030

if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) {

kernel/bpf/verifier.c

7031

verbose(env,

kernel/bpf/verifier.c

7036

if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) {

kernel/bpf/verifier.c

7037

verbose(env, "Acquiring the same lock again, AA deadlock detected\n");

kernel/bpf/verifier.c

7048

err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr);

kernel/bpf/verifier.c

7050

verbose(env, "Failed to acquire lock state\n");

kernel/bpf/verifier.c

7063

verbose(env, "%s_unlock without taking a lock\n", lock_str);

kernel/bpf/verifier.c

7074

verbose(env, "%s_unlock of different lock\n", lock_str);

kernel/bpf/verifier.c

7078

verbose(env, "%s_unlock cannot be out of order\n", lock_str);

kernel/bpf/verifier.c

7082

verbose(env, "%s_unlock of different lock\n", lock_str);

kernel/bpf/verifier.c

7086

invalidate_non_owning_refs(env);

kernel/bpf/verifier.c

7092

static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

7103

verbose(env,

kernel/bpf/verifier.c

7105

reg_arg_name(env, argno), struct_name);

kernel/bpf/verifier.c

7109

verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name,

kernel/bpf/verifier.c

7114

verbose(env, "map '%s' has no valid %s\n", map->name, struct_name);

kernel/bpf/verifier.c

7128

verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);

kernel/bpf/verifier.c

7132

verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n",

kernel/bpf/verifier.c

7137

verifier_bug(env, "Two map pointers in a %s helper", struct_name);

kernel/bpf/verifier.c

7145

static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

7149

verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n");

kernel/bpf/verifier.c

715

err = validate_ref_obj(env, ref_obj);

kernel/bpf/verifier.c

7152

return check_map_field_pointer(env, reg, argno, BPF_TIMER, map);

kernel/bpf/verifier.c

7155

static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

7158

return process_timer_func(env, reg, argno, &meta->map);

kernel/bpf/verifier.c

7161

static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

7164

return process_timer_func(env, reg, argno, &meta->map);

kernel/bpf/verifier.c

7167

static int process_kptr_func(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

7170

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

7181

verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",

kernel/bpf/verifier.c

7190

verbose(env,

kernel/bpf/verifier.c

7197

verbose(env, "R%d has no valid kptr\n", regno);

kernel/bpf/verifier.c

7204

verbose(env, "off=%d doesn't point to kptr\n", kptr_off);

kernel/bpf/verifier.c

7208

verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);

kernel/bpf/verifier.c

7238

static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

7245

verbose(env,

kernel/bpf/verifier.c

7247

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7268

if (!is_dynptr_reg_valid_uninit(env, reg)) {

kernel/bpf/verifier.c

7269

verbose(env, "Dynptr has to be an uninitialized dynptr\n");

kernel/bpf/verifier.c

7275

err = check_mem_access(env, insn_idx, reg, argno,

kernel/bpf/verifier.c

7281

err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr);

kernel/bpf/verifier.c

7285

verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n");

kernel/bpf/verifier.c

7289

if (!is_dynptr_reg_valid_init(env, reg)) {

kernel/bpf/verifier.c

7290

verbose(env, "Expected an initialized dynptr as %s\n",

kernel/bpf/verifier.c

7291

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7296

if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) {

kernel/bpf/verifier.c

7297

verbose(env,

kernel/bpf/verifier.c

730

id = acquire_reference(env, insn_idx, parent_id);

kernel/bpf/verifier.c

7300

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7305

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

7307

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

7315

mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS);

kernel/bpf/verifier.c

7358

static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx,

kernel/bpf/verifier.c

7361

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

7367

verbose(env, "%s expected pointer to an iterator on stack\n",

kernel/bpf/verifier.c

7368

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7380

verbose(env, "expected valid iter pointer as %s\n",

kernel/bpf/verifier.c

7381

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7389

if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {

kernel/bpf/verifier.c

7390

verbose(env, "expected uninitialized iter_%s as %s\n",

kernel/bpf/verifier.c

7391

iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));

kernel/bpf/verifier.c

7396

err = check_mem_access(env, insn_idx, reg, argno,

kernel/bpf/verifier.c

740

mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,

kernel/bpf/verifier.c

7402

err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);

kernel/bpf/verifier.c

7409

err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);

kernel/bpf/verifier.c

7414

verbose(env, "expected an initialized iter_%s as %s\n",

kernel/bpf/verifier.c

7415

iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));

kernel/bpf/verifier.c

7418

verbose(env, "expected an RCU CS when using %s\n", meta->func_name);

kernel/bpf/verifier.c

7424

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

7428

mark_stack_slots_scratched(env, spi, nr_slots);

kernel/bpf/verifier.c

7436

err = unmark_stack_slots_iter(env, reg, nr_slots);

kernel/bpf/verifier.c

7448

static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7457

head = bpf_explored_state(env, insn_idx);

kernel/bpf/verifier.c

746

static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack)

kernel/bpf/verifier.c

7482

static void maybe_widen_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7491

__mark_reg_unknown(env, rcur);

kernel/bpf/verifier.c

7494

static int widen_imprecise_scalars(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7506

maybe_widen_reg(env,

kernel/bpf/verifier.c

7517

maybe_widen_reg(env,

kernel/bpf/verifier.c

755

bpf_mark_reg_not_init(env, &stack[0].spilled_ptr);

kernel/bpf/verifier.c

756

bpf_mark_reg_not_init(env, &stack[1].spilled_ptr);

kernel/bpf/verifier.c

759

static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

761

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

7612

static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,

kernel/bpf/verifier.c

7615

struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;

kernel/bpf/verifier.c

7625

verifier_bug(env, "unexpected iterator state %d (%s)",

kernel/bpf/verifier.c

7636

verifier_bug(env, "bad parent state for iter next call");

kernel/bpf/verifier.c

764

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

7643

prev_st = find_prev_entry(env, cur_st->parent, insn_idx);

kernel/bpf/verifier.c

7645

queued_st = push_stack(env, insn_idx + 1, insn_idx, false);

kernel/bpf/verifier.c

7653

widen_imprecise_scalars(env, prev_st, queued_st);

kernel/bpf/verifier.c

7662

__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);

kernel/bpf/verifier.c

7689

static int resolve_map_arg_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7695

verifier_bug(env, "invalid map_ptr to access map->type");

kernel/bpf/verifier.c

7705

verbose(env, "invalid arg_type for sockmap/sockhash\n");

kernel/bpf/verifier.c

774

return release_reference(env, dynptr_type_referenced(reg->dynptr.type)

kernel/bpf/verifier.c

779

static void __mark_reg_unknown(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

782

static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

7833

static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,

kernel/bpf/verifier.c

784

if (!env->allow_ptr_leaks)

kernel/bpf/verifier.c

7844

verifier_bug(env, "unsupported arg type %d", arg_type);

kernel/bpf/verifier.c

785

bpf_mark_reg_not_init(env, reg);

kernel/bpf/verifier.c

787

__mark_reg_unknown(env, reg);

kernel/bpf/verifier.c

7882

verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7884

verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));

kernel/bpf/verifier.c

7885

verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));

kernel/bpf/verifier.c

7894

verbose(env,

kernel/bpf/verifier.c

7897

reg_arg_name(env, argno), reg_type_str(env, reg->type));

kernel/bpf/verifier.c

790

static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id)

kernel/bpf/verifier.c

7920

verbose(env, "Possibly NULL pointer passed to helper %s\n",

kernel/bpf/verifier.c

7921

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7927

verifier_bug(env, "missing arg compatible BTF ID");

kernel/bpf/verifier.c

7934

if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno)))

kernel/bpf/verifier.c

7938

verbose(env, "verifier internal error:");

kernel/bpf/verifier.c

7939

verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n",

kernel/bpf/verifier.c

7940

reg_arg_name(env, argno));

kernel/bpf/verifier.c

7944

err = __check_ptr_off_reg(env, reg, argno, true);

kernel/bpf/verifier.c

7948

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,

kernel/bpf/verifier.c

7951

verbose(env, "%s is of type %s but %s is expected\n",

kernel/bpf/verifier.c

7952

reg_arg_name(env, argno),

kernel/bpf/verifier.c

7966

verifier_bug(env, "unimplemented handling of MEM_ALLOC");

kernel/bpf/verifier.c

797

bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, 1 << STACK_DYNPTR, ({

kernel/bpf/verifier.c

7974

map_kptr_match_type(env, meta->kptr_field, reg, regno))

kernel/bpf/verifier.c

7984

verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match");

kernel/bpf/verifier.c

8007

static int __check_func_arg_reg_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8035

verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n",

kernel/bpf/verifier.c

8036

reg_arg_name(env, argno));

kernel/bpf/verifier.c

8071

return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok);

kernel/bpf/verifier.c

8079

if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog))

kernel/bpf/verifier.c

8083

return __check_ptr_off_reg(env, reg, argno, false);

kernel/bpf/verifier.c

8087

static int check_func_arg_reg_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

809

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8091

return __check_func_arg_reg_off(env, reg, argno, arg_type, true);

kernel/bpf/verifier.c

8094

static int check_arg_const_str(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8107

verbose(env, "%s points to insn_array map which cannot be used as const string\n",

kernel/bpf/verifier.c

8108

reg_arg_name(env, argno));

kernel/bpf/verifier.c

8113

verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

8118

verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

8123

verbose(env, "no direct value access support for this map type\n");

kernel/bpf/verifier.c

8127

err = check_map_access(env, reg, argno, 0,

kernel/bpf/verifier.c

8136

verbose(env, "direct value access on string failed\n");

kernel/bpf/verifier.c

8142

verbose(env, "string is not zero-terminated\n");

kernel/bpf/verifier.c

8149

static int get_constant_map_key(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8154

struct bpf_func_state *state = bpf_func(env, key);

kernel/bpf/verifier.c

8163

if (!env->bpf_capable)

kernel/bpf/verifier.c

8200

bpf_bt_set_frame_slot(&env->bt, key->frameno, spi);

kernel/bpf/verifier.c

8201

err = mark_chain_precision_batch(env, env->cur_state);

kernel/bpf/verifier.c

8211

static int check_func_arg(struct bpf_verifier_env *env, u32 arg,

kernel/bpf/verifier.c

8217

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8228

err = check_reg_arg(env, regno, SRC_OP);

kernel/bpf/verifier.c

8233

if (is_pointer_value(env, regno)) {

kernel/bpf/verifier.c

8234

verbose(env, "R%d leaks addr into helper function\n",

kernel/bpf/verifier.c

8242

!may_access_direct_pkt_data(env, meta, BPF_READ)) {

kernel/bpf/verifier.c

8243

verbose(env, "helper access to the packet is not allowed\n");

kernel/bpf/verifier.c

8248

err = resolve_map_arg_type(env, meta, &arg_type);

kernel/bpf/verifier.c

8264

err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta);

kernel/bpf/verifier.c

8268

err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type);

kernel/bpf/verifier.c

8274

!reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) {

kernel/bpf/verifier.c

8275

verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n",

kernel/bpf/verifier.c

8276

func_id_name(meta->func_id), reg_arg_name(env, argno));

kernel/bpf/verifier.c

8280

if (reg_is_referenced(env, reg))

kernel/bpf/verifier.c

8301

verbose(env,

kernel/bpf/verifier.c

832

dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) {

kernel/bpf/verifier.c

8321

verifier_bug(env, "invalid map_ptr to access map->key");

kernel/bpf/verifier.c

8325

err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL);

kernel/bpf/verifier.c

8329

err = get_constant_map_key(env, reg, key_size, &meta->const_map_key);

kernel/bpf/verifier.c

833

verbose(env, "cannot overwrite referenced dynptr\n");

kernel/bpf/verifier.c

8348

verifier_bug(env, "invalid map_ptr to access map->value");

kernel/bpf/verifier.c

8352

err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size,

kernel/bpf/verifier.c

8358

verbose(env, "Helper has invalid btf_id in R%d\n", regno);

kernel/bpf/verifier.c

8365

if (in_rbtree_lock_required_cb(env)) {

kernel/bpf/verifier.c

8366

verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");

kernel/bpf/verifier.c

8370

err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK);

kernel/bpf/verifier.c

8374

err = process_spin_lock(env, reg, argno_from_reg(regno), 0);

kernel/bpf/verifier.c

8378

verifier_bug(env, "spin lock arg on unexpected helper");

kernel/bpf/verifier.c

838

err = release_reference(env, state->stack[spi].spilled_ptr.id);

kernel/bpf/verifier.c

8383

err = process_timer_helper(env, reg, argno_from_reg(regno), meta);

kernel/bpf/verifier.c

8396

err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg],

kernel/bpf/verifier.c

840

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

8402

err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);

kernel/bpf/verifier.c

8406

err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),

kernel/bpf/verifier.c

841

mark_stack_slot_scratched(env, spi - 1);

kernel/bpf/verifier.c

8413

err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),

kernel/bpf/verifier.c

8420

err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj,

kernel/bpf/verifier.c

8427

verbose(env, "R%d is not a known constant'\n",

kernel/bpf/verifier.c

8432

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

8438

err = check_arg_const_str(env, reg, argno_from_reg(regno));

kernel/bpf/verifier.c

8444

err = process_kptr_func(env, regno, meta);

kernel/bpf/verifier.c

8453

static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)

kernel/bpf/verifier.c

8455

enum bpf_attach_type eatype = env->prog->expected_attach_type;

kernel/bpf/verifier.c

8456

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

847

static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

8487

verbose(env, "cannot update sockmap in this context\n");

kernel/bpf/verifier.c

8491

bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

8493

return env->prog->jit_requested &&

kernel/bpf/verifier.c

8497

static int check_map_func_compatibility(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

854

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

8573

!may_update_sockmap(env, func_id))

kernel/bpf/verifier.c

8582

!may_update_sockmap(env, func_id))

kernel/bpf/verifier.c

8636

if (env->subprog_cnt > 1 && !bpf_allow_tail_call_in_subprogs(env)) {

kernel/bpf/verifier.c

8637

verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");

kernel/bpf/verifier.c

8745

verbose(env, "cannot pass map_type %d into func %s#%d\n",

kernel/bpf/verifier.c

875

static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

877

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

8872

static void clear_all_pkt_pointers(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

8877

bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({

kernel/bpf/verifier.c

8879

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

889

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

8951

static int release_reference(struct bpf_verifier_env *env, int id)

kernel/bpf/verifier.c

8954

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

8955

struct bpf_idmap *idstack = &env->idmap_scratch;

kernel/bpf/verifier.c

8979

verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n",

kernel/bpf/verifier.c

8996

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

8998

invalidate_dynptr(env, stack);

kernel/bpf/verifier.c

9005

static void invalidate_non_owning_refs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

9010

bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({

kernel/bpf/verifier.c

9012

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

9016

static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

9023

bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({

kernel/bpf/verifier.c

9031

static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id)

kernel/bpf/verifier.c

9037

err = release_reference_nomark(env->cur_state, id);

kernel/bpf/verifier.c

9039

bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({

kernel/bpf/verifier.c

904

static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

9052

static void clear_caller_saved_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9059

bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);

kernel/bpf/verifier.c

9060

__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

9064

static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

907

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

9070

bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);

kernel/bpf/verifier.c

9073

typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9078

static int set_callee_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9082

static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,

kernel/bpf/verifier.c

9090

verbose(env, "the call stack of %d frames is too deep\n",

kernel/bpf/verifier.c

9096

verifier_bug(env, "Frame %d already allocated", state->curframe + 1);

kernel/bpf/verifier.c

9110

init_func_state(env, callee,

kernel/bpf/verifier.c

9115

err = set_callee_state_cb(env, caller, callee, callsite);

kernel/bpf/verifier.c

9130

static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

9134

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

9135

struct bpf_func_state *caller = cur_func(env);

kernel/bpf/verifier.c

9136

struct bpf_verifier_log *log = &env->log;

kernel/bpf/verifier.c

9141

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

9144

err = check_outgoing_stack_args(env, caller, sub->arg_cnt);

kernel/bpf/verifier.c

9151

ret = check_outgoing_stack_args(env, caller, sub->arg_cnt);

kernel/bpf/verifier.c

9165

bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno));

kernel/bpf/verifier.c

9175

ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX);

kernel/bpf/verifier.c

9183

reg_arg_name(env, argno));

kernel/bpf/verifier.c

9187

ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE);

kernel/bpf/verifier.c

919

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

9190

if (check_mem_reg(env, reg, argno, arg->mem_size))

kernel/bpf/verifier.c

9195

reg_arg_name(env, argno));

kernel/bpf/verifier.c

9208

reg_arg_name(env, argno));

kernel/bpf/verifier.c

9212

ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR);

kernel/bpf/verifier.c

9216

ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL);

kernel/bpf/verifier.c

9227

err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta);

kernel/bpf/verifier.c

9228

err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type);

kernel/bpf/verifier.c

9232

verifier_bug(env, "unrecognized %s type %d",

kernel/bpf/verifier.c

9233

reg_arg_name(env, argno), arg->arg_type);

kernel/bpf/verifier.c

9248

static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

9251

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

9266

err = btf_check_func_arg_match(env, subprog, btf, regs);

kernel/bpf/verifier.c

9276

static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

928

static bool in_rcu_cs(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

9280

struct bpf_verifier_state *state = env->cur_state, *callback_state;

kernel/bpf/verifier.c

9285

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

9293

env->subprog_info[subprog].is_cb = true;

kernel/bpf/verifier.c

9296

verifier_bug(env, "kfunc %s#%d not marked as callback-calling",

kernel/bpf/verifier.c

9301

verifier_bug(env, "helper %s#%d not marked as callback-calling",

kernel/bpf/verifier.c

9310

env->subprog_info[subprog].is_async_cb = true;

kernel/bpf/verifier.c

9311

async_cb = push_async_cb(env, env->subprog_info[subprog].start,

kernel/bpf/verifier.c

9313

is_async_cb_sleepable(env, insn));

kernel/bpf/verifier.c

932

static int mark_stack_slots_iter(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9320

err = set_callee_state_cb(env, caller, callee, insn_idx);

kernel/bpf/verifier.c

9330

callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);

kernel/bpf/verifier.c

9334

err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,

kernel/bpf/verifier.c

9345

static int process_bpf_exit_full(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9348

static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

9351

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

9358

subprog = bpf_find_subprog(env, target_insn);

kernel/bpf/verifier.c

9359

if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",

kernel/bpf/verifier.c

9364

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

9367

if (bpf_subprog_is_global(env, subprog)) {

kernel/bpf/verifier.c

9368

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

937

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

9370

if (env->cur_state->active_locks) {

kernel/bpf/verifier.c

9371

verbose(env, "global function calls are not allowed while holding a lock,\n"

kernel/bpf/verifier.c

9376

if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) {

kernel/bpf/verifier.c

9377

verbose(env, "sleepable global function %s() called in %s\n",

kernel/bpf/verifier.c

9378

sub_name, non_sleepable_context_description(env));

kernel/bpf/verifier.c

9383

verbose(env, "Caller passes invalid args into func#%d ('%s')\n",

kernel/bpf/verifier.c

9388

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

9389

verbose(env, "Func#%d ('%s') is global and assumed valid.\n",

kernel/bpf/verifier.c

9391

if (env->subprog_info[subprog].changes_pkt_data)

kernel/bpf/verifier.c

9392

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

9394

subprog_aux(env, subprog)->called = true;

kernel/bpf/verifier.c

9395

clear_caller_saved_regs(env, caller->regs);

kernel/bpf/verifier.c

9396

invalidate_outgoing_stack_args(env, cur_func(env));

kernel/bpf/verifier.c

9399

if (!subprog_returns_void(env, subprog)) {

kernel/bpf/verifier.c

940

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

9400

mark_reg_unknown(env, caller->regs, BPF_REG_0);

kernel/bpf/verifier.c

9404

if (env->subprog_info[subprog].might_throw) {

kernel/bpf/verifier.c

9407

branch = push_stack(env, *insn_idx + 1, *insn_idx, false);

kernel/bpf/verifier.c

9409

verbose(env, "failed to push state for global subprog exception path\n");

kernel/bpf/verifier.c

9412

return process_bpf_exit_full(env, NULL, true);

kernel/bpf/verifier.c

9423

caller_info = &env->subprog_info[caller->subprogno];

kernel/bpf/verifier.c

9424

callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);

kernel/bpf/verifier.c

9432

err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);

kernel/bpf/verifier.c

9436

clear_caller_saved_regs(env, caller->regs);

kernel/bpf/verifier.c

9439

*insn_idx = env->subprog_info[subprog].start - 1;

kernel/bpf/verifier.c

944

id = acquire_reference(env, insn_idx, 0);

kernel/bpf/verifier.c

9441

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

9442

verbose(env, "caller:\n");

kernel/bpf/verifier.c

9443

print_verifier_state(env, state, caller->frameno, true);

kernel/bpf/verifier.c

9444

verbose(env, "callee:\n");

kernel/bpf/verifier.c

9445

print_verifier_state(env, state, state->curframe, true);

kernel/bpf/verifier.c

9451

int map_set_for_each_callback_args(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9474

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9478

static int set_callee_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9492

static int set_map_elem_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9497

struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

9505

verbose(env, "callback function not allowed for map\n");

kernel/bpf/verifier.c

9509

err = map->ops->map_set_for_each_callback_args(env, caller, callee);

kernel/bpf/verifier.c

9518

static int set_loop_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9531

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

9532

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9533

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9540

static int set_timer_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

955

if (in_rcu_cs(env))

kernel/bpf/verifier.c

9563

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9564

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9570

static int set_find_vma_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9591

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9592

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9598

static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9607

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_0]);

kernel/bpf/verifier.c

9608

mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);

kernel/bpf/verifier.c

9612

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

9613

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9614

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9621

static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9642

ref_set_non_owning(env, &callee->regs[BPF_REG_1]);

kernel/bpf/verifier.c

9644

ref_set_non_owning(env, &callee->regs[BPF_REG_2]);

kernel/bpf/verifier.c

9646

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

9647

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9648

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

9654

static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9677

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

9678

bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

969

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

9690

static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

9692

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

9693

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

9717

static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)

kernel/bpf/verifier.c

9719

struct bpf_verifier_state *state = env->cur_state, *prev_st;

kernel/bpf/verifier.c

9734

verbose(env, "cannot return stack pointer to the caller\n");

kernel/bpf/verifier.c

9741

verbose(env, "R0 not a scalar value\n");

kernel/bpf/verifier.c

9746

err = mark_chain_precision(env, BPF_REG_0);

kernel/bpf/verifier.c

975

static int unmark_stack_slots_iter(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9752

verbose_invalid_scalar(env, r0, callee->callback_ret_range,

kernel/bpf/verifier.c

9756

if (!bpf_calls_callback(env, callee->callsite)) {

kernel/bpf/verifier.c

9757

verifier_bug(env, "in callback at %d, callsite %d !calls_callback",

kernel/bpf/verifier.c

9776

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

9777

verbose(env, "returning from callee:\n");

kernel/bpf/verifier.c

9778

print_verifier_state(env, state, callee->frameno, true);

kernel/bpf/verifier.c

9779

verbose(env, "to caller at %d:\n", *insn_idx);

kernel/bpf/verifier.c

978

struct bpf_func_state *state = bpf_func(env, reg);

kernel/bpf/verifier.c

9780

print_verifier_state(env, state, caller->frameno, true);

kernel/bpf/verifier.c

9786

invalidate_outgoing_stack_args(env, caller);

kernel/bpf/verifier.c

9799

prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;

kernel/bpf/verifier.c

9801

err = widen_imprecise_scalars(env, prev_st, state);

kernel/bpf/verifier.c

9808

static int do_refine_retval_range(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

981

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

9815

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

9841

env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT)

kernel/bpf/verifier.c

9845

env->prog->expected_attach_type == BPF_LSM_CGROUP) {

kernel/bpf/verifier.c

9846

if (!env->prog->aux->attach_func_proto->type)

kernel/bpf/verifier.c

9848

bpf_lsm_get_retval_range(env->prog, &range);

kernel/bpf/verifier.c

9860

return reg_bounds_sanity_check(env, ret_reg, "retval");

kernel/bpf/verifier.c

9864

record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,

kernel/bpf/verifier.c

9867

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

9883

verifier_bug(env, "expected map for helper call");

kernel/bpf/verifier.c

9896

verbose(env, "write into map forbidden\n");

kernel/bpf/verifier.c

990

WARN_ON_ONCE(release_reference(env, st->id));

kernel/bpf/verifier.c

9910

record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,

kernel/bpf/verifier.c

9913

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

992

bpf_mark_reg_not_init(env, st);

kernel/bpf/verifier.c

9922

verbose(env, "expected prog array map for tail call");

kernel/bpf/verifier.c

9926

reg = reg_state(env, BPF_REG_3);

kernel/bpf/verifier.c

9935

err = mark_chain_precision(env, BPF_REG_3);

kernel/bpf/verifier.c

9946

static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)

kernel/bpf/verifier.c

9948

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

9949

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

9950

struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);

kernel/bpf/verifier.c

9954

if (!exception_exit && cur_func(env)->frameno)

kernel/bpf/verifier.c

9966

verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",

kernel/bpf/verifier.c

997

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

9973

static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)

kernel/bpf/verifier.c

9977

if (check_lock && env->cur_state->active_locks) {

kernel/bpf/verifier.c

9978

verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);

kernel/bpf/verifier.c

9982

err = check_reference_leak(env, exception_exit);

kernel/bpf/verifier.c

9984

verbose(env, "%s would lead to reference leak\n", prefix);

kernel/bpf/verifier.c

9988

if (check_lock && env->cur_state->active_irq_id) {

kernel/bpf/verifier.c

9989

verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix);

kernel/bpf/verifier.c

9993

if (check_lock && env->cur_state->active_rcu_locks) {

kernel/bpf/verifier.c

9994

verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);

kernel/bpf/verifier.c

9998

if (check_lock && env->cur_state->active_preempt_locks) {

kernel/bpf/verifier.c

9999

verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);

kernel/sched/ext/ext.c

5068

static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

kernel/sched/ext/ext.c

5082

return add_uevent_var(env, "SCXOPS=%s", sch->ops.name);

kernel/sched/fair.c

10289

static int task_hot(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10293

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

10302

if (env->sd->flags & SD_SHARE_CPUCAPACITY)

kernel/sched/fair.c

10308

if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&

kernel/sched/fair.c

10319

if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))

kernel/sched/fair.c

10325

delta = rq_clock_task(env->src_rq) - p->se.exec_start;

kernel/sched/fair.c

10336

static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10345

if (!p->numa_faults || !(env->sd->flags & SD_NUMA))

kernel/sched/fair.c

10348

src_nid = cpu_to_node(env->src_cpu);

kernel/sched/fair.c

10349

dst_nid = cpu_to_node(env->dst_cpu);

kernel/sched/fair.c

10356

if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)

kernel/sched/fair.c

10367

if (env->idle == CPU_IDLE)

kernel/sched/fair.c

10384

struct lb_env *env)

kernel/sched/fair.c

10613

alb_break_llc(struct lb_env *env)

kernel/sched/fair.c

10618

if (cpus_share_cache(env->src_cpu, env->dst_cpu))

kernel/sched/fair.c

10628

if (env->src_rq->nr_pref_llc_running &&

kernel/sched/fair.c

10629

env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) {

kernel/sched/fair.c

10633

if (env->src_rq->nr_running <= 1)

kernel/sched/fair.c

10636

cur = rcu_dereference_all(env->src_rq->curr);

kernel/sched/fair.c

10640

if (can_migrate_llc(env->src_cpu, env->dst_cpu,

kernel/sched/fair.c

10652

static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10668

if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)

kernel/sched/fair.c

10676

if (env->migration_type == migrate_llc_task &&

kernel/sched/fair.c

10677

READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))

kernel/sched/fair.c

10680

if (can_migrate_llc_task(env->src_cpu,

kernel/sched/fair.c

10681

env->dst_cpu, p) != mig_forbid)

kernel/sched/fair.c

10695

alb_break_llc(struct lb_env *env)

kernel/sched/fair.c

10701

migrate_degrades_llc(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10710

int can_migrate_task(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10714

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

10727

if ((p->se.sched_delayed) && (env->migration_type != migrate_load))

kernel/sched/fair.c

10730

if (lb_throttled_hierarchy(p, env->dst_cpu))

kernel/sched/fair.c

10739

if (!env->sd->nr_balance_failed &&

kernel/sched/fair.c

10740

task_is_ineligible_on_dst_cpu(p, env->dst_cpu))

kernel/sched/fair.c

10750

if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {

kernel/sched/fair.c

10755

env->flags |= LBF_SOME_PINNED;

kernel/sched/fair.c

10767

if (env->idle == CPU_NEWLY_IDLE ||

kernel/sched/fair.c

10768

env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))

kernel/sched/fair.c

10772

cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr);

kernel/sched/fair.c

10775

env->flags |= LBF_DST_PINNED;

kernel/sched/fair.c

10776

env->new_dst_cpu = cpu;

kernel/sched/fair.c

10783

env->flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

10785

if (task_on_cpu(env->src_rq, p) ||

kernel/sched/fair.c

10786

task_current_donor(env->src_rq, p)) {

kernel/sched/fair.c

10798

if (env->flags & LBF_ACTIVE_LB)

kernel/sched/fair.c

10801

degrades = migrate_degrades_locality(p, env);

kernel/sched/fair.c

10808

if (migrate_degrades_llc(p, env)) {

kernel/sched/fair.c

10815

if (env->migration_type != migrate_llc_task)

kernel/sched/fair.c

10816

env->flags |= LBF_LLC_PINNED;

kernel/sched/fair.c

10821

hot = task_hot(p, env);

kernel/sched/fair.c

10826

if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) {

kernel/sched/fair.c

10839

static void detach_task(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

10841

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

10845

schedstat_inc(env->sd->lb_hot_gained[env->idle]);

kernel/sched/fair.c

10849

WARN_ON(task_current(env->src_rq, p));

kernel/sched/fair.c

10850

WARN_ON(task_current_donor(env->src_rq, p));

kernel/sched/fair.c

10852

deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);

kernel/sched/fair.c

10853

set_task_cpu(p, env->dst_cpu);

kernel/sched/fair.c

10862

static struct task_struct *detach_one_task(struct lb_env *env)

kernel/sched/fair.c

10866

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

10869

&env->src_rq->cfs_tasks, se.group_node) {

kernel/sched/fair.c

10870

if (!can_migrate_task(p, env))

kernel/sched/fair.c

10873

detach_task(p, env);

kernel/sched/fair.c

10881

schedstat_inc(env->sd->lb_gained[env->idle]);

kernel/sched/fair.c

10893

static int detach_tasks(struct lb_env *env)

kernel/sched/fair.c

10895

struct list_head *tasks = &env->src_rq->cfs_tasks;

kernel/sched/fair.c

10900

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

10906

if (env->src_rq->nr_running <= 1) {

kernel/sched/fair.c

10907

env->flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

10911

if (env->imbalance <= 0)

kernel/sched/fair.c

10919

if (env->idle && env->src_rq->nr_running <= 1)

kernel/sched/fair.c

10922

env->loop++;

kernel/sched/fair.c

10924

if (env->loop > env->loop_max)

kernel/sched/fair.c

10928

if (env->loop > env->loop_break) {

kernel/sched/fair.c

10929

env->loop_break += SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

10930

env->flags |= LBF_NEED_BREAK;

kernel/sched/fair.c

10936

if (!can_migrate_task(p, env))

kernel/sched/fair.c

10939

switch (env->migration_type) {

kernel/sched/fair.c

10951

load < 16 && !env->sd->nr_balance_failed)

kernel/sched/fair.c

10960

if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)

kernel/sched/fair.c

10963

env->imbalance -= load;

kernel/sched/fair.c

10969

if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance)

kernel/sched/fair.c

10972

env->imbalance -= util;

kernel/sched/fair.c

10976

env->imbalance--;

kernel/sched/fair.c

10981

if (task_fits_cpu(p, env->src_cpu))

kernel/sched/fair.c

10984

env->imbalance = 0;

kernel/sched/fair.c

10988

env->imbalance--;

kernel/sched/fair.c

10992

detach_task(p, env);

kernel/sched/fair.c

10993

list_add(&p->se.group_node, &env->tasks);

kernel/sched/fair.c

11003

if (env->idle == CPU_NEWLY_IDLE)

kernel/sched/fair.c

11011

if (env->imbalance <= 0)

kernel/sched/fair.c

11027

schedstat_add(env->sd->lb_gained[env->idle], detached);

kernel/sched/fair.c

11036

static void attach_tasks(struct lb_env *env)

kernel/sched/fair.c

11038

struct list_head *tasks = &env->tasks;

kernel/sched/fair.c

11042

rq_lock(env->dst_rq, &rf);

kernel/sched/fair.c

11043

update_rq_clock(env->dst_rq);

kernel/sched/fair.c

11049

attach_task(env->dst_rq, p);

kernel/sched/fair.c

11052

rq_unlock(env->dst_rq, &rf);

kernel/sched/fair.c

11599

sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group)

kernel/sched/fair.c

11609

return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu));

kernel/sched/fair.c

11623

static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs,

kernel/sched/fair.c

11626

if (!env->idle)

kernel/sched/fair.c

11642

static inline long sibling_imbalance(struct lb_env *env,

kernel/sched/fair.c

11650

if (!env->idle || !busiest->sum_nr_running)

kernel/sched/fair.c

11696

static void record_sg_llc_stats(struct lb_env *env,

kernel/sched/fair.c

11703

if (!sched_cache_enabled() || env->idle == CPU_NEWLY_IDLE)

kernel/sched/fair.c

11707

if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu)))

kernel/sched/fair.c

11735

static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,

kernel/sched/fair.c

11741

if (env->sd->flags & SD_SHARE_LLC)

kernel/sched/fair.c

11749

if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)

kernel/sched/fair.c

11754

env->dst_cpu, 0, true) == mig_llc)

kernel/sched/fair.c

11760

static bool update_llc_busiest(struct lb_env *env,

kernel/sched/fair.c

11770

static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs,

kernel/sched/fair.c

11775

static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,

kernel/sched/fair.c

11781

static bool update_llc_busiest(struct lb_env *env,

kernel/sched/fair.c

11797

static inline void update_sg_lb_stats(struct lb_env *env,

kernel/sched/fair.c

11803

int i, nr_running, local_group, sd_flags = env->sd->flags;

kernel/sched/fair.c

11804

bool balancing_at_rd = !env->sd->parent;

kernel/sched/fair.c

11810

for_each_cpu_and(i, sched_group_span(group), env->cpus) {

kernel/sched/fair.c

11830

dst_llc = llc_id(env->dst_cpu);

kernel/sched/fair.c

11868

} else if (env->idle && sched_reduced_capacity(rq, env->sd)) {

kernel/sched/fair.c

11881

if (env->idle && sgs->sum_h_nr_running &&

kernel/sched/fair.c

11882

sched_group_asym(env, sgs, group))

kernel/sched/fair.c

11886

if (smt_balance(env, sgs, group))

kernel/sched/fair.c

11890

if (llc_balance(env, sgs, group))

kernel/sched/fair.c

11894

sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);

kernel/sched/fair.c

11896

record_sg_llc_stats(env, sgs, group);

kernel/sched/fair.c

11916

static bool update_sd_pick_busiest(struct lb_env *env,

kernel/sched/fair.c

11938

if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&

kernel/sched/fair.c

11940

(!env->dst_core_idle ||

kernel/sched/fair.c

11941

!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||

kernel/sched/fair.c

11963

return update_llc_busiest(env, busiest, sgs);

kernel/sched/fair.c

12057

if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&

kernel/sched/fair.c

12059

(capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))

kernel/sched/fair.c

12428

static void update_idle_cpu_scan(struct lb_env *env,

kernel/sched/fair.c

12432

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

12443

if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)

kernel/sched/fair.c

12503

static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)

kernel/sched/fair.c

12505

struct sched_group *sg = env->sd->groups;

kernel/sched/fair.c

12511

env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu);

kernel/sched/fair.c

12517

local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));

kernel/sched/fair.c

12522

if (env->idle != CPU_NEWLY_IDLE ||

kernel/sched/fair.c

12524

update_group_capacity(env->sd, env->dst_cpu);

kernel/sched/fair.c

12527

update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded);

kernel/sched/fair.c

12529

if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {

kernel/sched/fair.c

12542

} while (sg != env->sd->groups);

kernel/sched/fair.c

12553

if (env->sd->flags & SD_NUMA)

kernel/sched/fair.c

12554

env->fbq_type = fbq_classify_group(&sds->busiest_stat);

kernel/sched/fair.c

12556

if (!env->sd->parent) {

kernel/sched/fair.c

12558

set_rd_overloaded(env->dst_rq->rd, sg_overloaded);

kernel/sched/fair.c

12561

set_rd_overutilized(env->dst_rq->rd, sg_overutilized);

kernel/sched/fair.c

12563

set_rd_overutilized(env->dst_rq->rd, sg_overutilized);

kernel/sched/fair.c

12566

update_idle_cpu_scan(env, sum_util);

kernel/sched/fair.c

12575

static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)

kernel/sched/fair.c

12583

if (env->sd->flags & SD_ASYM_CPUCAPACITY) {

kernel/sched/fair.c

12585

env->migration_type = migrate_misfit;

kernel/sched/fair.c

12586

env->imbalance = 1;

kernel/sched/fair.c

12592

env->migration_type = migrate_load;

kernel/sched/fair.c

12593

env->imbalance = busiest->group_misfit_task_load;

kernel/sched/fair.c

12603

env->migration_type = migrate_task;

kernel/sched/fair.c

12604

env->imbalance = busiest->sum_h_nr_running;

kernel/sched/fair.c

12610

env->migration_type = migrate_task;

kernel/sched/fair.c

12611

env->imbalance = 1;

kernel/sched/fair.c

12618

env->migration_type = migrate_llc_task;

kernel/sched/fair.c

12619

env->imbalance = 1;

kernel/sched/fair.c

12631

env->migration_type = migrate_task;

kernel/sched/fair.c

12632

env->imbalance = 1;

kernel/sched/fair.c

12642

!(env->sd->flags & SD_SHARE_LLC)) {

kernel/sched/fair.c

12651

env->migration_type = migrate_util;

kernel/sched/fair.c

12652

env->imbalance = max(local->group_capacity, local->group_util) -

kernel/sched/fair.c

12662

if (env->idle && env->imbalance == 0) {

kernel/sched/fair.c

12663

env->migration_type = migrate_task;

kernel/sched/fair.c

12664

env->imbalance = 1;

kernel/sched/fair.c

12675

env->migration_type = migrate_task;

kernel/sched/fair.c

12676

env->imbalance = sibling_imbalance(env, sds, busiest, local);

kernel/sched/fair.c

12683

env->migration_type = migrate_task;

kernel/sched/fair.c

12684

env->imbalance = max_t(long, 0,

kernel/sched/fair.c

12690

if (env->sd->flags & SD_NUMA) {

kernel/sched/fair.c

12691

env->imbalance = adjust_numa_imbalance(env->imbalance,

kernel/sched/fair.c

12693

env->sd->imb_numa_nr);

kernel/sched/fair.c

12698

env->imbalance >>= 1;

kernel/sched/fair.c

12721

env->imbalance = 0;

kernel/sched/fair.c

12733

env->imbalance = 0;

kernel/sched/fair.c

12747

env->migration_type = migrate_load;

kernel/sched/fair.c

12748

env->imbalance = min(

kernel/sched/fair.c

12786

static struct sched_group *sched_balance_find_src_group(struct lb_env *env)

kernel/sched/fair.c

12797

update_sd_lb_stats(env, &sds);

kernel/sched/fair.c

12809

if (!is_rd_overutilized(env->dst_rq->rd) &&

kernel/sched/fair.c

12810

rcu_dereference_all(env->dst_rq->rd->pd))

kernel/sched/fair.c

12861

env->sd->imbalance_pct * local->avg_load)

kernel/sched/fair.c

12871

sibling_imbalance(env, &sds, busiest, local) > 1))

kernel/sched/fair.c

12875

if (!env->idle) {

kernel/sched/fair.c

12914

calculate_imbalance(env, &sds);

kernel/sched/fair.c

12915

return env->imbalance ? sds.busiest : NULL;

kernel/sched/fair.c

12918

env->imbalance = 0;

kernel/sched/fair.c

12925

static struct rq *sched_balance_find_src_rq(struct lb_env *env,

kernel/sched/fair.c

12936

for_each_cpu_and(i, sched_group_span(group), env->cpus) {

kernel/sched/fair.c

12963

if (rt > env->fbq_type)

kernel/sched/fair.c

12978

if (env->sd->flags & SD_ASYM_CPUCAPACITY &&

kernel/sched/fair.c

12979

!capacity_greater(capacity_of(env->dst_cpu), capacity) &&

kernel/sched/fair.c

12990

if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1)

kernel/sched/fair.c

12993

switch (env->migration_type) {

kernel/sched/fair.c

13001

if (nr_running == 1 && load > env->imbalance &&

kernel/sched/fair.c

13002

!check_cpu_capacity(rq, env->sd))

kernel/sched/fair.c

13064

dst_llc = llc_id(env->dst_cpu);

kernel/sched/fair.c

13091

asym_active_balance(struct lb_env *env)

kernel/sched/fair.c

13103

return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) &&

kernel/sched/fair.c

13104

(sched_asym_prefer(env->dst_cpu, env->src_cpu) ||

kernel/sched/fair.c

13105

!sched_use_asym_prio(env->sd, env->src_cpu));

kernel/sched/fair.c

13109

imbalanced_active_balance(struct lb_env *env)

kernel/sched/fair.c

13111

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

13118

if ((env->migration_type == migrate_task) &&

kernel/sched/fair.c

13125

static int need_active_balance(struct lb_env *env)

kernel/sched/fair.c

13127

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

13129

if (alb_break_llc(env))

kernel/sched/fair.c

13132

if (asym_active_balance(env))

kernel/sched/fair.c

13135

if (imbalanced_active_balance(env))

kernel/sched/fair.c

13144

if (env->idle &&

kernel/sched/fair.c

13145

(env->src_rq->cfs.h_nr_runnable == 1)) {

kernel/sched/fair.c

13146

if ((check_cpu_capacity(env->src_rq, sd)) &&

kernel/sched/fair.c

13147

(capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))

kernel/sched/fair.c

13151

if (env->migration_type == migrate_misfit ||

kernel/sched/fair.c

13152

env->migration_type == migrate_llc_task)

kernel/sched/fair.c

13160

static int should_we_balance(struct lb_env *env)

kernel/sched/fair.c

13163

struct sched_group *sg = env->sd->groups;

kernel/sched/fair.c

13170

if (!cpumask_test_cpu(env->dst_cpu, env->cpus))

kernel/sched/fair.c

13180

if (env->idle == CPU_NEWLY_IDLE) {

kernel/sched/fair.c

13181

if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)

kernel/sched/fair.c

13188

for_each_cpu_and(cpu, swb_cpus, env->cpus) {

kernel/sched/fair.c

13198

!(env->sd->flags & SD_SHARE_CPUCAPACITY) &&

kernel/sched/fair.c

13215

return cpu == env->dst_cpu;

kernel/sched/fair.c

13220

return idle_smt == env->dst_cpu;

kernel/sched/fair.c

13223

return group_balance_cpu(sg) == env->dst_cpu;

kernel/sched/fair.c

13226

static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd,

kernel/sched/fair.c

13232

switch (env->migration_type) {

kernel/sched/fair.c

13234

__schedstat_add(sd->lb_imbalance_load[idle], env->imbalance);

kernel/sched/fair.c

13237

__schedstat_add(sd->lb_imbalance_util[idle], env->imbalance);

kernel/sched/fair.c

13240

__schedstat_add(sd->lb_imbalance_task[idle], env->imbalance);

kernel/sched/fair.c

13243

__schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);

kernel/sched/fair.c

13279

struct lb_env env = {

kernel/sched/fair.c

13288

.tasks = LIST_HEAD_INIT(env.tasks),

kernel/sched/fair.c

13297

if (!should_we_balance(&env)) {

kernel/sched/fair.c

13310

group = sched_balance_find_src_group(&env);

kernel/sched/fair.c

13316

busiest = sched_balance_find_src_rq(&env, group);

kernel/sched/fair.c

13322

WARN_ON_ONCE(busiest == env.dst_rq);

kernel/sched/fair.c

13324

update_lb_imbalance_stat(&env, sd, idle);

kernel/sched/fair.c

13326

env.src_cpu = busiest->cpu;

kernel/sched/fair.c

13327

env.src_rq = busiest;

kernel/sched/fair.c

13331

env.flags |= LBF_ALL_PINNED;

kernel/sched/fair.c

13339

env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);

kernel/sched/fair.c

13349

cur_ld_moved = detach_tasks(&env);

kernel/sched/fair.c

13362

attach_tasks(&env);

kernel/sched/fair.c

13368

if (env.flags & LBF_NEED_BREAK) {

kernel/sched/fair.c

13369

env.flags &= ~LBF_NEED_BREAK;

kernel/sched/fair.c

13392

if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {

kernel/sched/fair.c

13395

__cpumask_clear_cpu(env.dst_cpu, env.cpus);

kernel/sched/fair.c

13397

env.dst_rq = cpu_rq(env.new_dst_cpu);

kernel/sched/fair.c

13398

env.dst_cpu = env.new_dst_cpu;

kernel/sched/fair.c

13399

env.flags &= ~LBF_DST_PINNED;

kernel/sched/fair.c

13400

env.loop = 0;

kernel/sched/fair.c

13401

env.loop_break = SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

13416

if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)

kernel/sched/fair.c

13421

if (unlikely(env.flags & LBF_ALL_PINNED)) {

kernel/sched/fair.c

13431

if (!cpumask_subset(cpus, env.dst_grpmask)) {

kernel/sched/fair.c

13432

env.loop = 0;

kernel/sched/fair.c

13433

env.loop_break = SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

13458

env.migration_type != migrate_misfit &&

kernel/sched/fair.c

13459

!(env.flags & LBF_LLC_PINNED))

kernel/sched/fair.c

13462

if (need_active_balance(&env)) {

kernel/sched/fair.c

13478

env.flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

13504

if (likely(!active_balance) || need_active_balance(&env)) {

kernel/sched/fair.c

13517

if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {

kernel/sched/fair.c

13547

if (env.idle == CPU_NEWLY_IDLE ||

kernel/sched/fair.c

13548

env.migration_type == migrate_misfit)

kernel/sched/fair.c

13552

if ((env.flags & LBF_ALL_PINNED &&

kernel/sched/fair.c

13649

struct lb_env env = {

kernel/sched/fair.c

13662

p = detach_one_task(&env);

kernel/sched/fair.c

2877

static void update_numa_stats(struct task_numa_env *env,

kernel/sched/fair.c

2898

!cpumask_test_cpu(cpu, env->p->cpus_ptr))

kernel/sched/fair.c

2911

ns->node_type = numa_classify(env->imbalance_pct, ns);

kernel/sched/fair.c

2917

static void task_numa_assign(struct task_numa_env *env,

kernel/sched/fair.c

2920

struct rq *rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2923

if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {

kernel/sched/fair.c

2925

int start = env->dst_cpu;

kernel/sched/fair.c

2928

for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {

kernel/sched/fair.c

2929

if (cpu == env->best_cpu || !idle_cpu(cpu) ||

kernel/sched/fair.c

2930

!cpumask_test_cpu(cpu, env->p->cpus_ptr)) {

kernel/sched/fair.c

2934

env->dst_cpu = cpu;

kernel/sched/fair.c

2935

rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2949

if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {

kernel/sched/fair.c

2950

rq = cpu_rq(env->best_cpu);

kernel/sched/fair.c

2954

if (env->best_task)

kernel/sched/fair.c

2955

put_task_struct(env->best_task);

kernel/sched/fair.c

2959

env->best_task = p;

kernel/sched/fair.c

2960

env->best_imp = imp;

kernel/sched/fair.c

2961

env->best_cpu = env->dst_cpu;

kernel/sched/fair.c

2965

struct task_numa_env *env)

kernel/sched/fair.c

2978

src_capacity = env->src_stats.compute_capacity;

kernel/sched/fair.c

2979

dst_capacity = env->dst_stats.compute_capacity;

kernel/sched/fair.c

2983

orig_src_load = env->src_stats.load;

kernel/sched/fair.c

2984

orig_dst_load = env->dst_stats.load;

kernel/sched/fair.c

3005

static bool task_numa_compare(struct task_numa_env *env,

kernel/sched/fair.c

3008

struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);

kernel/sched/fair.c

3009

struct rq *dst_rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

3013

int dist = env->dist;

kernel/sched/fair.c

3031

if (cur == env->p) {

kernel/sched/fair.c

3037

if (maymove && moveimp >= env->best_imp)

kernel/sched/fair.c

3044

if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))

kernel/sched/fair.c

3051

if (env->best_task &&

kernel/sched/fair.c

3052

env->best_task->numa_preferred_nid == env->src_nid &&

kernel/sched/fair.c

3053

cur->numa_preferred_nid != env->src_nid) {

kernel/sched/fair.c

3075

if (env->dst_stats.node_type == node_has_spare)

kernel/sched/fair.c

3078

imp = taskimp + task_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

3079

task_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

3092

imp += group_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

3093

group_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

3095

imp += task_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

3096

task_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

3100

if (cur->numa_preferred_nid == env->dst_nid)

kernel/sched/fair.c

3109

if (cur->numa_preferred_nid == env->src_nid)

kernel/sched/fair.c

3112

if (maymove && moveimp > imp && moveimp > env->best_imp) {

kernel/sched/fair.c

3122

if (env->best_task && cur->numa_preferred_nid == env->src_nid &&

kernel/sched/fair.c

3123

env->best_task->numa_preferred_nid != env->src_nid) {

kernel/sched/fair.c

3133

if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)

kernel/sched/fair.c

3139

load = task_h_load(env->p) - task_h_load(cur);

kernel/sched/fair.c

3143

dst_load = env->dst_stats.load + load;

kernel/sched/fair.c

3144

src_load = env->src_stats.load - load;

kernel/sched/fair.c

3146

if (load_too_imbalanced(src_load, dst_load, env))

kernel/sched/fair.c

3152

int cpu = env->dst_stats.idle_cpu;

kernel/sched/fair.c

3156

cpu = env->dst_cpu;

kernel/sched/fair.c

3162

if (!idle_cpu(cpu) && env->best_cpu >= 0 &&

kernel/sched/fair.c

3163

idle_cpu(env->best_cpu)) {

kernel/sched/fair.c

3164

cpu = env->best_cpu;

kernel/sched/fair.c

3167

env->dst_cpu = cpu;

kernel/sched/fair.c

3170

task_numa_assign(env, cur, imp);

kernel/sched/fair.c

3177

if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))

kernel/sched/fair.c

3184

if (!maymove && env->best_task &&

kernel/sched/fair.c

3185

env->best_task->numa_preferred_nid == env->src_nid) {

kernel/sched/fair.c

3194

static void task_numa_find_cpu(struct task_numa_env *env,

kernel/sched/fair.c

3204

if (env->dst_stats.node_type == node_has_spare) {

kernel/sched/fair.c

3214

src_running = env->src_stats.nr_running - 1;

kernel/sched/fair.c

3215

dst_running = env->dst_stats.nr_running + 1;

kernel/sched/fair.c

3218

env->imb_numa_nr);

kernel/sched/fair.c

3223

if (env->dst_stats.idle_cpu >= 0) {

kernel/sched/fair.c

3224

env->dst_cpu = env->dst_stats.idle_cpu;

kernel/sched/fair.c

3225

task_numa_assign(env, NULL, 0);

kernel/sched/fair.c

3235

load = task_h_load(env->p);

kernel/sched/fair.c

3236

dst_load = env->dst_stats.load + load;

kernel/sched/fair.c

3237

src_load = env->src_stats.load - load;

kernel/sched/fair.c

3238

maymove = !load_too_imbalanced(src_load, dst_load, env);

kernel/sched/fair.c

3242

for_each_cpu_and(cpu, cpumask_of_node(env->dst_nid), env->p->cpus_ptr) {

kernel/sched/fair.c

3243

env->dst_cpu = cpu;

kernel/sched/fair.c

3244

if (task_numa_compare(env, taskimp, groupimp, maymove))

kernel/sched/fair.c

3251

struct task_numa_env env = {

kernel/sched/fair.c

3279

sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu));

kernel/sched/fair.c

3281

env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;

kernel/sched/fair.c

3282

env.imb_numa_nr = sd->imb_numa_nr;

kernel/sched/fair.c

3297

env.dst_nid = p->numa_preferred_nid;

kernel/sched/fair.c

3298

dist = env.dist = node_distance(env.src_nid, env.dst_nid);

kernel/sched/fair.c

3299

taskweight = task_weight(p, env.src_nid, dist);

kernel/sched/fair.c

3300

groupweight = group_weight(p, env.src_nid, dist);

kernel/sched/fair.c

3301

update_numa_stats(&env, &env.src_stats, env.src_nid, false);

kernel/sched/fair.c

3302

taskimp = task_weight(p, env.dst_nid, dist) - taskweight;

kernel/sched/fair.c

3303

groupimp = group_weight(p, env.dst_nid, dist) - groupweight;

kernel/sched/fair.c

3304

update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);

kernel/sched/fair.c

3307

task_numa_find_cpu(&env, taskimp, groupimp);

kernel/sched/fair.c

3317

if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {

kernel/sched/fair.c

3319

if (nid == env.src_nid || nid == p->numa_preferred_nid)

kernel/sched/fair.c

3322

dist = node_distance(env.src_nid, env.dst_nid);

kernel/sched/fair.c

3324

dist != env.dist) {

kernel/sched/fair.c

3325

taskweight = task_weight(p, env.src_nid, dist);

kernel/sched/fair.c

3326

groupweight = group_weight(p, env.src_nid, dist);

kernel/sched/fair.c

3335

env.dist = dist;

kernel/sched/fair.c

3336

env.dst_nid = nid;

kernel/sched/fair.c

3337

update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);

kernel/sched/fair.c

3338

task_numa_find_cpu(&env, taskimp, groupimp);

kernel/sched/fair.c

3351

if (env.best_cpu == -1)

kernel/sched/fair.c

3352

nid = env.src_nid;

kernel/sched/fair.c

3354

nid = cpu_to_node(env.best_cpu);

kernel/sched/fair.c

3361

if (env.best_cpu == -1) {

kernel/sched/fair.c

3362

trace_sched_stick_numa(p, env.src_cpu, NULL, -1);

kernel/sched/fair.c

3366

best_rq = cpu_rq(env.best_cpu);

kernel/sched/fair.c

3367

if (env.best_task == NULL) {

kernel/sched/fair.c

3368

ret = migrate_task_to(p, env.best_cpu);

kernel/sched/fair.c

3371

trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);

kernel/sched/fair.c

3375

ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);

kernel/sched/fair.c

3379

trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);

kernel/sched/fair.c

3380

put_task_struct(env.best_task);

kernel/trace/rv/monitors/nomiss/nomiss.c

38

static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)

kernel/trace/rv/monitors/nomiss/nomiss.c

40

if (env == clk_nomiss)

kernel/trace/rv/monitors/nomiss/nomiss.c

41

return ha_get_clk_ns(ha_mon, env, time_ns);

kernel/trace/rv/monitors/nomiss/nomiss.c

42

else if (env == is_constr_dl_nomiss)

kernel/trace/rv/monitors/nomiss/nomiss.c

44

else if (env == is_defer_nomiss)

kernel/trace/rv/monitors/nomiss/nomiss.c

49

static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)

kernel/trace/rv/monitors/nomiss/nomiss.c

51

if (env == clk_nomiss)

kernel/trace/rv/monitors/nomiss/nomiss.c

52

ha_reset_clk_ns(ha_mon, env, time_ns);

kernel/trace/rv/monitors/nomiss/nomiss_trace.h

17

TP_PROTO(int id, char *state, char *event, char *env),

kernel/trace/rv/monitors/nomiss/nomiss_trace.h

18

TP_ARGS(id, state, event, env));

kernel/trace/rv/monitors/opid/opid.c

20

static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_opid env, u64 time_ns)

kernel/trace/rv/monitors/opid/opid.c

22

if (env == irq_off_opid)

kernel/trace/rv/monitors/opid/opid.c

24

else if (env == preempt_off_opid) {

kernel/trace/rv/monitors/opid/opid_trace.h

17

TP_PROTO(char *state, char *event, char *env),

kernel/trace/rv/monitors/opid/opid_trace.h

18

TP_ARGS(state, event, env));

kernel/trace/rv/monitors/stall/stall.c

23

static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_stall env, u64 time_ns)

kernel/trace/rv/monitors/stall/stall.c

25

if (env == clk_stall)

kernel/trace/rv/monitors/stall/stall.c

26

return ha_get_clk_jiffy(ha_mon, env);

kernel/trace/rv/monitors/stall/stall.c

30

static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_stall env, u64 time_ns)

kernel/trace/rv/monitors/stall/stall.c

32

if (env == clk_stall)

kernel/trace/rv/monitors/stall/stall.c

33

ha_reset_clk_jiffy(ha_mon, env);

kernel/trace/rv/monitors/stall/stall_trace.h

17

TP_PROTO(int id, char *state, char *event, char *env),

kernel/trace/rv/monitors/stall/stall_trace.h

18

TP_ARGS(id, state, event, env));

kernel/trace/rv/rv_trace.h

165

TP_PROTO(int id, char *state, char *event, char *env),

kernel/trace/rv/rv_trace.h

167

TP_ARGS(id, state, event, env),

kernel/trace/rv/rv_trace.h

173

__string( env, env )

kernel/trace/rv/rv_trace.h

179

__assign_str(env);

kernel/trace/rv/rv_trace.h

187

__get_str(env))

kernel/trace/rv/rv_trace.h

71

TP_PROTO(char *state, char *event, char *env),

kernel/trace/rv/rv_trace.h

73

TP_ARGS(state, event, env),

kernel/trace/rv/rv_trace.h

78

__string( env, env )

kernel/trace/rv/rv_trace.h

84

__assign_str(env);

kernel/trace/rv/rv_trace.h

90

__get_str(env))

lib/kobject_uevent.c

116

struct kobj_uevent_env *env = NULL;

lib/kobject_uevent.c

127

env = kzalloc_obj(*env);

lib/kobject_uevent.c

128

if (!env)

lib/kobject_uevent.c

133

add_uevent_var(env, "SYNTH_UUID=%.*s", UUID_STRING_LEN, buf))

lib/kobject_uevent.c

167

if (add_uevent_var(env, "SYNTH_ARG_%.*s=%.*s",

lib/kobject_uevent.c

175

kfree(env);

lib/kobject_uevent.c

177

*ret_env = env;

lib/kobject_uevent.c

196

struct kobj_uevent_env *env;

lib/kobject_uevent.c

212

count - (action_args - buf), &env);

lib/kobject_uevent.c

221

r = kobject_uevent_env(kobj, action, env->envp);

lib/kobject_uevent.c

222

kfree(env);

lib/kobject_uevent.c

251

static int init_uevent_argv(struct kobj_uevent_env *env, const char *subsystem)

lib/kobject_uevent.c

253

int buffer_size = sizeof(env->buf) - env->buflen;

lib/kobject_uevent.c

256

len = strscpy(&env->buf[env->buflen], subsystem, buffer_size);

lib/kobject_uevent.c

263

env->argv[0] = uevent_helper;

lib/kobject_uevent.c

264

env->argv[1] = &env->buf[env->buflen];

lib/kobject_uevent.c

265

env->argv[2] = NULL;

lib/kobject_uevent.c

267

env->buflen += len + 1;

lib/kobject_uevent.c

278

static struct sk_buff *alloc_uevent_skb(struct kobj_uevent_env *env,

lib/kobject_uevent.c

289

skb = alloc_skb(len + env->buflen, GFP_KERNEL);

lib/kobject_uevent.c

297

skb_put_data(skb, env->buf, env->buflen);

lib/kobject_uevent.c

308

static int uevent_net_broadcast_untagged(struct kobj_uevent_env *env,

lib/kobject_uevent.c

326

skb = alloc_uevent_skb(env, action_string, devpath);

lib/kobject_uevent.c

344

struct kobj_uevent_env *env,

lib/kobject_uevent.c

352

skb = alloc_uevent_skb(env, action_string, devpath);

lib/kobject_uevent.c

383

struct kobj_uevent_env *env,

lib/kobject_uevent.c

410

ret = uevent_net_broadcast_untagged(env, action_string,

lib/kobject_uevent.c

415

ret = uevent_net_broadcast_tagged(net->uevent_sock->sk, env,

lib/kobject_uevent.c

423

static void zap_modalias_env(struct kobj_uevent_env *env)

lib/kobject_uevent.c

429

for (i = 0; i < env->envp_idx;) {

lib/kobject_uevent.c

430

if (strncmp(env->envp[i], modalias_prefix,

lib/kobject_uevent.c

436

len = strlen(env->envp[i]) + 1;

lib/kobject_uevent.c

438

if (i != env->envp_idx - 1) {

lib/kobject_uevent.c

454

memmove(env->envp[i], env->envp[i + 1],

lib/kobject_uevent.c

455

env->buflen - (env->envp[i + 1] - env->envp[0]));

lib/kobject_uevent.c

457

for (j = i; j < env->envp_idx - 1; j++)

lib/kobject_uevent.c

458

env->envp[j] = env->envp[j + 1] - len;

lib/kobject_uevent.c

461

env->envp_idx--;

lib/kobject_uevent.c

462

env->buflen -= len;

lib/kobject_uevent.c

479

struct kobj_uevent_env *env;

lib/kobject_uevent.c

543

env = kzalloc_obj(struct kobj_uevent_env);

lib/kobject_uevent.c

544

if (!env)

lib/kobject_uevent.c

555

retval = add_uevent_var(env, "ACTION=%s", action_string);

lib/kobject_uevent.c

558

retval = add_uevent_var(env, "DEVPATH=%s", devpath);

lib/kobject_uevent.c

561

retval = add_uevent_var(env, "SUBSYSTEM=%s", subsystem);

lib/kobject_uevent.c

568

retval = add_uevent_var(env, "%s", envp_ext[i]);

lib/kobject_uevent.c

576

retval = uevent_ops->uevent(kobj, env);

lib/kobject_uevent.c

598

zap_modalias_env(env);

lib/kobject_uevent.c

606

retval = add_uevent_var(env, "SEQNUM=%llu",

lib/kobject_uevent.c

611

retval = kobject_uevent_net_broadcast(kobj, env, action_string,

lib/kobject_uevent.c

619

retval = add_uevent_var(env, "HOME=/");

lib/kobject_uevent.c

622

retval = add_uevent_var(env,

lib/kobject_uevent.c

626

retval = init_uevent_argv(env, subsystem);

lib/kobject_uevent.c

631

info = call_usermodehelper_setup(env->argv[0], env->argv,

lib/kobject_uevent.c

632

env->envp, GFP_KERNEL,

lib/kobject_uevent.c

633

NULL, cleanup_uevent_env, env);

lib/kobject_uevent.c

636

env = NULL; /* freed by cleanup_uevent_env */

lib/kobject_uevent.c

643

kfree(env);

lib/kobject_uevent.c

671

int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...)

lib/kobject_uevent.c

676

if (env->envp_idx >= ARRAY_SIZE(env->envp)) {

lib/kobject_uevent.c

682

len = vsnprintf(&env->buf[env->buflen],

lib/kobject_uevent.c

683

sizeof(env->buf) - env->buflen,

lib/kobject_uevent.c

687

if (len >= (sizeof(env->buf) - env->buflen)) {

lib/kobject_uevent.c

692

env->envp[env->envp_idx++] = &env->buf[env->buflen];

lib/kobject_uevent.c

693

env->buflen += len + 1;

net/atm/atm_sysfs.c

86

static int atm_uevent(const struct device *cdev, struct kobj_uevent_env *env)

net/atm/atm_sysfs.c

95

if (add_uevent_var(env, "NAME=%s%d", adev->type, adev->number))

net/core/net-sysfs.c

2197

static int netdev_uevent(const struct device *d, struct kobj_uevent_env *env)

net/core/net-sysfs.c

2203

retval = add_uevent_var(env, "INTERFACE=%s", dev->name);

net/core/net-sysfs.c

2211

retval = add_uevent_var(env, "IFINDEX=%d", dev->ifindex);

net/mac80211/parse.c

237

const struct ieee80211_tx_pwr_env *env = (const void *)data;

net/mac80211/parse.c

244

count = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_COUNT);

net/mac80211/parse.c

245

interpret = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_INTERPRET);

net/mac80211/parse.c

246

category = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_CATEGORY);

net/mac80211/parse.c

278

memcpy(out, env->variable, count + 1);

net/mac80211/parse.c

281

if (count == 3 && len > sizeof(*env) + count + 1) {

net/mac80211/parse.c

282

out[4] = env->variable[4];

net/mac80211/parse.c

289

memset(out, env->variable[0],

net/mac80211/parse.c

296

memcpy(out, env->variable, N);

net/mac80211/parse.c

300

if (len > sizeof(*env) + N) {

net/mac80211/parse.c

301

int K = u8_get_bits(env->variable[N],

net/mac80211/parse.c

305

memcpy(out + N, env->variable + N + 1, K);

net/rfkill/core.c

867

static int rfkill_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

net/rfkill/core.c

875

error = add_uevent_var(env, "RFKILL_NAME=%s", rfkill->name);

net/rfkill/core.c

878

error = add_uevent_var(env, "RFKILL_TYPE=%s",

net/rfkill/core.c

886

error = add_uevent_var(env, "RFKILL_STATE=%d",

net/rfkill/core.c

890

return add_uevent_var(env, "RFKILL_HW_BLOCK_REASON=0x%lx", reasons);

net/wireless/core.h

56

enum environment_cap env;

net/wireless/reg.c

3351

enum environment_cap env = ENVIRON_ANY;

net/wireless/reg.c

3369

env = ENVIRON_INDOOR;

net/wireless/reg.c

3371

env = ENVIRON_OUTDOOR;

net/wireless/reg.c

3392

request->country_ie_env = env;

net/wireless/reg.c

557

char *env[] = { country, NULL };

net/wireless/reg.c

574

ret = kobject_uevent_env(&reg_fdev->dev.kobj, KOBJ_CHANGE, env);

scripts/kconfig/conf.c

189

char *env = getenv("KCONFIG_PROBABILITY");

scripts/kconfig/conf.c

192

while (env && *env) {

scripts/kconfig/conf.c

194

int tmp = strtol(env, &endp, 10);

scripts/kconfig/conf.c

203

env = (*endp == ':') ? endp + 1 : endp;

scripts/kconfig/conf.c

90

char *env;

scripts/kconfig/conf.c

93

env = getenv("KCONFIG_SEED");

scripts/kconfig/conf.c

94

if (env && *env) {

scripts/kconfig/conf.c

97

seed = strtol(env, &endp, 0);

scripts/kconfig/confdata.c

216

const char *env = getenv("KCONFIG_WARN_CHANGED_INPUT");

scripts/kconfig/confdata.c

218

return env && *env;

scripts/kconfig/confdata.c

406

char *env;

scripts/kconfig/confdata.c

414

env = getenv("KCONFIG_DEFCONFIG_LIST");

scripts/kconfig/confdata.c

415

if (!env)

scripts/kconfig/confdata.c

421

while (isspace(*env))

scripts/kconfig/confdata.c

422

env++;

scripts/kconfig/confdata.c

424

if (!*env)

scripts/kconfig/confdata.c

427

p = env;

scripts/kconfig/confdata.c

435

name = env;

scripts/kconfig/confdata.c

447

env = p + 1;

scripts/kconfig/confdata.c

898

char *env;

scripts/kconfig/confdata.c

920

env = getenv("KCONFIG_OVERWRITECONFIG");

scripts/kconfig/confdata.c

921

if (env && *env) {

scripts/kconfig/gconf.c

1283

char *env = getenv(SRCTREE);

scripts/kconfig/gconf.c

1284

gchar *path = g_strconcat(env ? env : g_get_current_dir(), "/scripts/kconfig/icons/menu.xpm", NULL);

scripts/kconfig/gconf.c

1308

char *env;

scripts/kconfig/gconf.c

1315

env = getenv(SRCTREE);

scripts/kconfig/gconf.c

1316

if (env)

scripts/kconfig/gconf.c

1317

glade_file = g_strconcat(env, "/scripts/kconfig/gconf.ui", NULL);

scripts/kconfig/gconf.c

959

char *env = getenv(SRCTREE);

scripts/kconfig/gconf.c

960

gchar *path = g_strconcat(env ? env : g_get_current_dir(), "/scripts/kconfig/icons/", filename, NULL);

scripts/kconfig/preprocess.c

48

struct env *e;

scripts/kconfig/preprocess.c

57

static void env_del(struct env *e)

scripts/kconfig/preprocess.c

68

struct env *e;

scripts/kconfig/preprocess.c

94

struct env *e, *tmp;

security/tomoyo/common.c

1583

tomoyo_set_string(head, ptr->env->name);

security/tomoyo/common.h

1003

int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env) __must_hold_shared(&tomoyo_ss);

security/tomoyo/common.h

759

const struct tomoyo_path_info *env; /* environment variable */

security/tomoyo/environ.c

103

e.env = tomoyo_get_name(data);

security/tomoyo/environ.c

104

if (!e.env)

security/tomoyo/environ.c

108

tomoyo_put_name(e.env);

security/tomoyo/environ.c

24

return tomoyo_path_matches_pattern(r->param.environ.name, acl->env);

security/tomoyo/environ.c

51

int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env)

security/tomoyo/environ.c

56

if (!env || !*env)

security/tomoyo/environ.c

58

environ.name = env;

security/tomoyo/environ.c

83

return p1->env == p2->env;

security/tomoyo/gc.c

210

tomoyo_put_name(entry->env);

sound/aoa/soundbus/core.c

101

retval = add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);

sound/aoa/soundbus/core.c

104

retval = add_uevent_var(env, "MODALIAS=%s", soundbus_dev->modalias);

sound/aoa/soundbus/core.c

60

static int soundbus_uevent(const struct device *dev, struct kobj_uevent_env *env)

sound/aoa/soundbus/core.c

78

retval = add_uevent_var(env, "OF_NAME=%pOFn", of->dev.of_node);

sound/aoa/soundbus/core.c

82

retval = add_uevent_var(env, "OF_TYPE=%s", of_node_get_device_type(of->dev.of_node));

sound/aoa/soundbus/core.c

92

int tmp = env->buflen;

sound/aoa/soundbus/core.c

93

retval = add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);

sound/aoa/soundbus/core.c

96

compat += env->buflen - tmp;

sound/aoa/soundbus/core.c

97

cplen -= env->buflen - tmp;

sound/hda/core/hda_bus_type.c

67

static int hda_uevent(const struct device *dev, struct kobj_uevent_env *env)

sound/hda/core/hda_bus_type.c

73

if (add_uevent_var(env, "MODALIAS=%s", modalias))

tools/lib/subcmd/exec-cmd.c

121

char *env;

tools/lib/subcmd/exec-cmd.c

126

env = getenv(subcmd_config.exec_path_env);

tools/lib/subcmd/exec-cmd.c

127

if (env && *env)

tools/lib/subcmd/exec-cmd.c

128

return strdup(env);

tools/lib/subcmd/run-command.c

117

if (cmd->env) {

tools/lib/subcmd/run-command.c

118

for (; *cmd->env; cmd->env++) {

tools/lib/subcmd/run-command.c

119

if (strchr(*cmd->env, '='))

tools/lib/subcmd/run-command.c

120

putenv((char*)*cmd->env);

tools/lib/subcmd/run-command.c

122

unsetenv(*cmd->env);

tools/lib/subcmd/run-command.h

43

const char *const *env;

tools/objtool/builtin-check.c

117

char *env;

tools/objtool/builtin-check.c

120

env = getenv("OBJTOOL_ARGS");

tools/objtool/builtin-check.c

121

if (env) {

tools/objtool/builtin-check.c

124

envv[envc++] = env;

tools/objtool/builtin-check.c

125

env = strchr(env, ' ');

tools/objtool/builtin-check.c

126

if (!env)

tools/objtool/builtin-check.c

128

*env = '\0';

tools/objtool/builtin-check.c

129

env++;

tools/objtool/builtin-check.c

135

env = getenv("OBJTOOL_VERBOSE");

tools/objtool/builtin-check.c

136

if (env && !strcmp(env, "1"))

tools/perf/arch/common.c

114

char *env = getenv("PATH");

tools/perf/arch/common.c

116

if (!env)

tools/perf/arch/common.c

119

env = strdup(env);

tools/perf/arch/common.c

120

if (!env)

tools/perf/arch/common.c

123

path = strtok_r(env, ":", &tmp);

tools/perf/arch/common.c

132

free(env);

tools/perf/arch/common.c

149

static bool is_native_compatible(struct perf_env *env, uint16_t target, uint16_t host)

tools/perf/arch/common.c

166

bool target_is_64 = perf_env__kernel_is_64_bit(env);

tools/perf/arch/common.c

177

static int perf_env__lookup_binutils_path(struct perf_env *env,

tools/perf/arch/common.c

181

uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);

tools/perf/arch/common.c

190

if (is_native_compatible(env, e_machine, EM_HOST))

tools/perf/arch/common.c

222

path_list = perf_env__kernel_is_64_bit(env) ? riscv64_triplets : riscv32_triplets;

tools/perf/arch/common.c

242

ui__error("binutils for %s not supported.\n", perf_env__arch(env));

tools/perf/arch/common.c

251

name, perf_env__arch(env), name);

tools/perf/arch/common.c

267

int perf_env__lookup_objdump(struct perf_env *env, char **path)

tools/perf/arch/common.c

273

if (env->arch == NULL)

tools/perf/arch/common.c

276

return perf_env__lookup_binutils_path(env, "objdump", path);

tools/perf/arch/common.c

284

bool perf_env__single_address_space(struct perf_env *env)

tools/perf/arch/common.c

286

uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);

tools/perf/arch/common.h

10

bool perf_env__single_address_space(struct perf_env *env);

tools/perf/arch/common.h

9

int perf_env__lookup_objdump(struct perf_env *env, char **path);

tools/perf/builtin-c2c.c

2031

struct perf_env *env __maybe_unused)

tools/perf/builtin-c2c.c

2045

static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env)

tools/perf/builtin-c2c.c

2052

return sort_dimension__add(hpp_list, name, /*evlist=*/NULL, env, /*level=*/0);

tools/perf/builtin-c2c.c

207

struct perf_env *env);

tools/perf/builtin-c2c.c

2076

ret = _fn(hpp_list, tok, env); \

tools/perf/builtin-c2c.c

2090

struct perf_env *env)

tools/perf/builtin-c2c.c

2122

struct perf_env *env)

tools/perf/builtin-c2c.c

213

struct perf_env *env)

tools/perf/builtin-c2c.c

2136

return hpp_list__parse(&hists->list, /*output=*/NULL, sort, env);

tools/perf/builtin-c2c.c

2142

struct perf_env *env)

tools/perf/builtin-c2c.c

2145

return hpp_list__parse(&c2c_hists->list, output, sort, env);

tools/perf/builtin-c2c.c

227

ret = c2c_hists__init(hists, sort, nr_header_lines, env);

tools/perf/builtin-c2c.c

2280

struct perf_env *env = arg;

tools/perf/builtin-c2c.c

2294

c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env);

tools/perf/builtin-c2c.c

2338

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-c2c.c

2343

c2c.nodes_cnt = env->nr_numa_nodes;

tools/perf/builtin-c2c.c

2344

c2c.cpus_cnt = env->nr_cpus_avail;

tools/perf/builtin-c2c.c

2346

n = env->numa_nodes;

tools/perf/builtin-c2c.c

2525

static void print_pareto(FILE *out, struct perf_env *env)

tools/perf/builtin-c2c.c

2550

ret = hpp_list__parse(&hpp_list, cl_output, /*evlist=*/NULL, env);

tools/perf/builtin-c2c.c

3155

struct perf_env *env;

tools/perf/builtin-c2c.c

3225

env = perf_session__env(session);

tools/perf/builtin-c2c.c

3232

if (perf_env__e_machine(env, /*e_flags=*/NULL) == EM_AARCH64)

tools/perf/builtin-c2c.c

3262

err = mem2node__init(&c2c.mem2node, env);

tools/perf/builtin-c2c.c

3302

if (symbol__init(env) < 0)

tools/perf/builtin-c2c.c

412

c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2, machine->env);

tools/perf/builtin-record.c

2266

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-record.c

2273

env->clock.clockid_res_ns = rec->opts.clockid_res_ns;

tools/perf/builtin-record.c

2275

env->clock.clockid = rec->opts.clockid;

tools/perf/builtin-record.c

2290

env->clock.tod_ns = ref;

tools/perf/builtin-record.c

2295

env->clock.clockid_ns = ref;

tools/perf/builtin-record.c

2441

struct perf_env *env;

tools/perf/builtin-record.c

2483

env = perf_session__env(session);

tools/perf/builtin-record.c

2517

env->comp_type = PERF_COMP_ZSTD;

tools/perf/builtin-record.c

2518

env->comp_level = rec->opts.comp_level;

tools/perf/builtin-record.c

2571

env->comp_mmap_len = session->evlist->core.mmap_len;

tools/perf/builtin-record.c

2905

env->comp_ratio = ratio + 0.5;

tools/perf/builtin-sched.c

3385

struct perf_env *env;

tools/perf/builtin-sched.c

3411

env = perf_session__env(session);

tools/perf/builtin-sched.c

3420

symbol__init(env);

tools/perf/builtin-sched.c

3459

sched->max_cpu.cpu = min(env->nr_cpus_online, MAX_CPUS);

tools/perf/builtin-sched.c

4715

cd_map = session->header.env.cpu_domain;

tools/perf/builtin-sched.c

4717

session->header.env.nr_cpus_avail,

tools/perf/builtin-sched.c

4773

cd_map0 = session[0]->header.env.cpu_domain;

tools/perf/builtin-sched.c

4792

cd_map1 = session[1]->header.env.cpu_domain;

tools/perf/builtin-sched.c

4809

ret = show_schedstat_data(&cpu_head_ses0, cd_map0, session[0]->header.env.nr_cpus_avail,

tools/perf/builtin-sched.c

4810

&cpu_head_ses1, cd_map1, session[1]->header.env.nr_cpus_avail, true);

tools/perf/builtin-script.c

2501

struct cgroup *cgrp = cgroup__find(machine->env,

tools/perf/builtin-script.c

4212

struct perf_env *env;

tools/perf/builtin-script.c

4461

env = perf_session__env(session);

tools/perf/builtin-script.c

4471

if (symbol__init(env) < 0)

tools/perf/builtin-script.c

751

struct perf_env *env;

tools/perf/builtin-script.c

761

env = perf_session__env(script->session);

tools/perf/builtin-script.c

762

if (!env->clock.enabled) {

tools/perf/builtin-script.c

767

clockid_ns = env->clock.clockid_ns;

tools/perf/builtin-script.c

768

tod_ns = env->clock.tod_ns;

tools/perf/builtin-stat.c

1638

struct perf_env *env = data;

tools/perf/builtin-stat.c

1640

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1650

struct perf_env *env = data;

tools/perf/builtin-stat.c

1652

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1667

static void perf_env__get_cache_id_for_cpu(struct perf_cpu cpu, struct perf_env *env,

tools/perf/builtin-stat.c

1671

int caches_cnt = env->caches_cnt;

tools/perf/builtin-stat.c

1672

struct cpu_cache_level *caches = env->caches;

tools/perf/builtin-stat.c

1707

struct perf_env *env = data;

tools/perf/builtin-stat.c

1709

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1716

perf_env__get_cache_id_for_cpu(cpu, env, cache_level, &id);

tools/perf/builtin-stat.c

1725

struct perf_env *env = data;

tools/perf/builtin-stat.c

1727

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1740

struct perf_env *env = data;

tools/perf/builtin-stat.c

1742

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1760

struct perf_env *env = data;

tools/perf/builtin-stat.c

1762

struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);

tools/perf/builtin-stat.c

1900

struct perf_env *env = perf_session__env(st->session);

tools/perf/builtin-stat.c

1924

get_id, env, needs_sort);

tools/perf/builtin-stat.c

2216

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-stat.c

2217

const char **argv = env->cmdline_argv;

tools/perf/builtin-stat.c

2218

int argc = env->nr_cmdline;

tools/perf/builtin-timechart.c

1565

tchart->numcpus = ph->env.nr_cpus_avail;

tools/perf/builtin-timechart.c

1574

if (svg_build_topology_map(&ph->env))

tools/perf/builtin-top.c

832

!symbol__is_idle(al.sym, al.map ? map__dso(al.map) : NULL, machine->env)) {

tools/perf/tests/hists_cumulate.c

297

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

444

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

502

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

686

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_filter.c

146

if (setup_sorting(evlist, machine->env) < 0)

tools/perf/tests/hists_link.c

318

if (setup_sorting(evlist, machine->env) < 0)

tools/perf/tests/hists_output.c

148

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

250

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

306

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

380

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

482

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/mem2node.c

51

struct perf_env env = {

tools/perf/tests/mem2node.c

66

T("failed: mem2node__init", !mem2node__init(&map, &env));

tools/perf/tests/topology.c

106

e_machine = perf_env__e_machine(env, NULL);

tools/perf/tests/topology.c

108

if (!env->cpu && e_machine != EM_S390 && e_machine != EM_AARCH64)

tools/perf/tests/topology.c

122

TEST_ASSERT_VAL("Session header CPU map not set", env->cpu);

tools/perf/tests/topology.c

124

for (i = 0; i < (unsigned int)env->nr_cpus_avail; i++) {

tools/perf/tests/topology.c

129

env->cpu[i].core_id,

tools/perf/tests/topology.c

130

env->cpu[i].socket_id);

tools/perf/tests/topology.c

140

env->cpu[cpu.cpu].core_id == id.core);

tools/perf/tests/topology.c

142

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

145

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

154

env->cpu[cpu.cpu].core_id == id.core);

tools/perf/tests/topology.c

157

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

160

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

169

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

172

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

184

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

77

struct perf_env *env;

tools/perf/tests/topology.c

82

env = perf_session__env(session);

tools/perf/ui/browsers/hists.c

2295

struct perf_env *env)

tools/perf/ui/browsers/hists.c

2301

browser->env = env;

tools/perf/ui/browsers/hists.c

2475

perf_env__lookup_objdump(browser->env, &annotate_opts.objdump_path))

tools/perf/ui/browsers/hists.c

3001

struct perf_env *env, bool warn_lost_event)

tools/perf/ui/browsers/hists.c

3004

struct hist_browser *browser = perf_evsel_browser__new(evsel, hbt, env);

tools/perf/ui/browsers/hists.c

3236

if (env->arch)

tools/perf/ui/browsers/hists.c

3460

struct perf_env *env;

tools/perf/ui/browsers/hists.c

3557

menu->min_pcnt, menu->env,

tools/perf/ui/browsers/hists.c

3615

struct hist_browser_timer *hbt, float min_pcnt, struct perf_env *env,

tools/perf/ui/browsers/hists.c

3630

.env = env,

tools/perf/ui/browsers/hists.c

3665

float min_pcnt, struct perf_env *env, bool warn_lost_event)

tools/perf/ui/browsers/hists.c

3674

env, warn_lost_event);

tools/perf/ui/browsers/hists.c

3691

return __evlist__tui_browse_hists(evlist, nr_entries, help, hbt, min_pcnt, env,

tools/perf/ui/browsers/hists.c

3711

float min_percent, struct perf_env *env)

tools/perf/ui/browsers/hists.c

3729

browser->env = env;

tools/perf/ui/browsers/hists.h

16

struct perf_env *env;

tools/perf/util/amd-sample-raw.c

403

static void parse_cpuid(struct perf_env *env)

tools/perf/util/amd-sample-raw.c

408

cpuid = perf_env__cpuid(env);

tools/perf/util/amd-sample-raw.c

424

struct perf_env *env = perf_session__env(evlist->session);

tools/perf/util/amd-sample-raw.c

425

int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env);

tools/perf/util/amd-sample-raw.c

426

const char *pmu_mapping = perf_env__pmu_mappings(env);

tools/perf/util/amd-sample-raw.c

441

if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions"))

tools/perf/util/amd-sample-raw.c

444

if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat"))

tools/perf/util/amd-sample-raw.c

447

if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize"))

tools/perf/util/amd-sample-raw.c

450

if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket"))

tools/perf/util/amd-sample-raw.c

453

if (perf_env__find_pmu_cap(env, "ibs_op", "strmst"))

tools/perf/util/amd-sample-raw.c

458

parse_cpuid(env);

tools/perf/util/annotate.c

999

arch = arch__find(e_machine, e_flags, machine->env ? machine->env->cpuid : NULL);

tools/perf/util/aslr.c

364

struct perf_env *env = session_machine ? session_machine->env : NULL;

tools/perf/util/aslr.c

365

bool is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8);

tools/perf/util/auxtrace.c

1945

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/auxtrace.c

1955

int kernel_is_64_bit = perf_env__kernel_is_64_bit(env);

tools/perf/util/auxtrace.c

2057

struct perf_env *env, const struct perf_tool *tool,

tools/perf/util/auxtrace.c

2060

return __auxtrace_mmap__read(map, itr, env, tool, fn, false, 0);

tools/perf/util/auxtrace.c

2064

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/auxtrace.c

2068

return __auxtrace_mmap__read(map, itr, env, tool, fn, true, snapshot_size);

tools/perf/util/auxtrace.h

510

struct perf_env *env, const struct perf_tool *tool,

tools/perf/util/auxtrace.h

514

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/block-info.c

503

struct evsel *evsel, struct perf_env *env)

tools/perf/util/block-info.c

515

ret = block_hists_tui_browse(bh, evsel, min_percent, env);

tools/perf/util/block-info.h

72

struct evsel *evsel, struct perf_env *env);

tools/perf/util/bpf-event.c

108

static int perf_env__fetch_btf(struct perf_env *env,

tools/perf/util/bpf-event.c

126

if (!perf_env__insert_btf(env, node)) {

tools/perf/util/bpf-event.c

48

struct perf_env *env = machine->env;

tools/perf/util/bpf-event.c

526

struct perf_env *env = &session->header.env;

tools/perf/util/bpf-event.c

533

perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb,

tools/perf/util/bpf-event.c

54

if (env == NULL)

tools/perf/util/bpf-event.c

562

struct perf_env *env;

tools/perf/util/bpf-event.c

57

info_node = perf_env__find_bpf_prog_info(env, id);

tools/perf/util/bpf-event.c

571

env = perf_session__env(session);

tools/perf/util/bpf-event.c

623

perf_env__fetch_btf(env, info->btf_id, btf);

tools/perf/util/bpf-event.c

678

if (!perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/bpf-event.c

77

dso__bpf_prog(dso)->env = env;

tools/perf/util/bpf-event.c

853

static int perf_env__add_bpf_info(struct perf_env *env, u32 id)

tools/perf/util/bpf-event.c

888

if (!perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/bpf-event.c

910

perf_env__fetch_btf(env, btf_id, btf);

tools/perf/util/bpf-event.c

921

struct perf_env *env = data;

tools/perf/util/bpf-event.c

929

ret = perf_env__add_bpf_info(env, event->bpf.id);

tools/perf/util/bpf-event.c

946

int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)

tools/perf/util/bpf-event.c

963

return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);

tools/perf/util/bpf-event.c

967

struct perf_env *env,

tools/perf/util/bpf-event.c

994

node = __perf_env__find_btf(env, info->btf_id);

tools/perf/util/bpf-event.h

42

int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env);

tools/perf/util/bpf-event.h

44

struct perf_env *env,

tools/perf/util/bpf-event.h

56

struct perf_env *env __maybe_unused)

tools/perf/util/bpf-event.h

62

struct perf_env *env __maybe_unused,

tools/perf/util/cgroup.c

574

struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,

tools/perf/util/cgroup.c

579

down_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

580

cgrp = __cgroup__findnew(&env->cgroups.tree, id, true, path);

tools/perf/util/cgroup.c

581

up_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

590

struct cgroup *cgroup__find(struct perf_env *env, uint64_t id)

tools/perf/util/cgroup.c

594

down_read(&env->cgroups.lock);

tools/perf/util/cgroup.c

595

cgrp = __cgroup__findnew(&env->cgroups.tree, id, false, NULL);

tools/perf/util/cgroup.c

596

up_read(&env->cgroups.lock);

tools/perf/util/cgroup.c

600

void perf_env__purge_cgroups(struct perf_env *env)

tools/perf/util/cgroup.c

605

down_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

606

while (!RB_EMPTY_ROOT(&env->cgroups.tree)) {

tools/perf/util/cgroup.c

607

node = rb_first(&env->cgroups.tree);

tools/perf/util/cgroup.c

610

rb_erase(node, &env->cgroups.tree);

tools/perf/util/cgroup.c

613

up_write(&env->cgroups.lock);

tools/perf/util/cgroup.h

37

struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,

tools/perf/util/cgroup.h

39

struct cgroup *cgroup__find(struct perf_env *env, uint64_t id);

tools/perf/util/cgroup.h

42

void perf_env__purge_cgroups(struct perf_env *env);

tools/perf/util/data-convert-bt.c

1372

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1379

ncpus = env->nr_cpus_avail ?: MAX_CPUS;

tools/perf/util/data-convert-bt.c

1405

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1414

ADD("host", env->hostname);

tools/perf/util/data-convert-bt.c

1416

ADD("release", perf_env__os_release(env));

tools/perf/util/data-convert-bt.c

1417

ADD("version", env->version);

tools/perf/util/data-convert-bt.c

1418

ADD("machine", env->arch);

tools/perf/util/data-convert-bt.c

1452

if (session->header.env.hostname) {

tools/perf/util/data-convert-bt.c

1454

session->header.env.hostname);

tools/perf/util/data-convert-bt.c

1458

if (session->header.env.os_release) {

tools/perf/util/data-convert-bt.c

1460

session->header.env.os_release);

tools/perf/util/data-convert-bt.c

1464

if (session->header.env.version) {

tools/perf/util/data-convert-bt.c

1466

session->header.env.version);

tools/perf/util/data-convert-bt.c

1470

if (session->header.env.arch) {

tools/perf/util/data-convert-bt.c

1472

session->header.env.arch);

tools/perf/util/data-convert-bt.c

1510

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1512

if (!env->clock.enabled) {

tools/perf/util/data-convert-bt.c

1518

desc = clockid_name(env->clock.clockid);

tools/perf/util/data-convert-bt.c

1519

offset = env->clock.tod_ns - env->clock.clockid_ns;

tools/perf/util/data-convert-json.c

277

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-json.c

300

output_json_key_string(out, true, 2, "hostname", env->hostname);

tools/perf/util/data-convert-json.c

302

perf_env__os_release(env));

tools/perf/util/data-convert-json.c

303

output_json_key_string(out, true, 2, "arch", env->arch);

tools/perf/util/data-convert-json.c

305

if (env->cpu_desc)

tools/perf/util/data-convert-json.c

306

output_json_key_string(out, true, 2, "cpu-desc", env->cpu_desc);

tools/perf/util/data-convert-json.c

308

output_json_key_string(out, true, 2, "cpuid", env->cpuid);

tools/perf/util/data-convert-json.c

309

output_json_key_format(out, true, 2, "nrcpus-online", "%u", env->nr_cpus_online);

tools/perf/util/data-convert-json.c

310

output_json_key_format(out, true, 2, "nrcpus-avail", "%u", env->nr_cpus_avail);

tools/perf/util/data-convert-json.c

312

if (env->clock.enabled) {

tools/perf/util/data-convert-json.c

314

"%u", env->clock.clockid);

tools/perf/util/data-convert-json.c

316

"%" PRIu64, env->clock.clockid_ns);

tools/perf/util/data-convert-json.c

318

"%" PRIu64, env->clock.tod_ns);

tools/perf/util/data-convert-json.c

321

output_json_key_string(out, true, 2, "perf-version", env->version);

tools/perf/util/data-convert-json.c

324

for (i = 0; i < env->nr_cmdline; i++) {

tools/perf/util/data-convert-json.c

326

output_json_string(c->out, env->cmdline_argv[i]);

tools/perf/util/dso.c

1346

machine && machine->env ? perf_env__arch(machine->env) : NULL);

tools/perf/util/dso.c

1348

return perf_env__e_machine(machine ? machine->env : NULL, e_flags);

tools/perf/util/dso.c

2003

info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env,

tools/perf/util/dso.c

883

node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);

tools/perf/util/dso.c

911

node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);

tools/perf/util/dso.h

247

struct perf_env *env;

tools/perf/util/env.c

1000

for (i = 0; i < env->nr_cpu_pmu_caps; i++) {

tools/perf/util/env.c

1001

if (!strncmp(env->cpu_pmu_caps[i], cap_eq, cap_size + 1)) {

tools/perf/util/env.c

1003

return &env->cpu_pmu_caps[i][cap_size + 1];

tools/perf/util/env.c

1009

for (i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/env.c

101

down_read(&env->bpf_progs.lock);

tools/perf/util/env.c

1010

if (strcmp(env->pmu_caps[i].pmu_name, pmu_name))

tools/perf/util/env.c

1013

ptr = env->pmu_caps[i].caps;

tools/perf/util/env.c

1015

for (j = 0; j < env->pmu_caps[i].nr_caps; j++) {

tools/perf/util/env.c

102

first = rb_first(&env->bpf_progs.infos);

tools/perf/util/env.c

1028

void perf_env__find_br_cntr_info(struct perf_env *env,

tools/perf/util/env.c

1033

*nr = env->cpu_pmu_caps ? env->br_cntr_nr :

tools/perf/util/env.c

1034

env->pmu_caps->br_cntr_nr;

tools/perf/util/env.c

1038

*width = env->cpu_pmu_caps ? env->br_cntr_width :

tools/perf/util/env.c

1039

env->pmu_caps->br_cntr_width;

tools/perf/util/env.c

1043

bool perf_env__is_x86_amd_cpu(struct perf_env *env)

tools/perf/util/env.c

1048

is_amd = env->cpuid && strstarts(env->cpuid, "AuthenticAMD") ? 1 : -1;

tools/perf/util/env.c

105

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

1055

struct perf_env env = { .total_mem = 0, };

tools/perf/util/env.c

1058

perf_env__init(&env);

tools/perf/util/env.c

1059

perf_env__cpuid(&env);

tools/perf/util/env.c

1060

is_amd = perf_env__is_x86_amd_cpu(&env);

tools/perf/util/env.c

1061

perf_env__exit(&env);

tools/perf/util/env.c

1066

bool perf_env__is_x86_intel_cpu(struct perf_env *env)

tools/perf/util/env.c

1071

is_intel = env->cpuid && strstarts(env->cpuid, "GenuineIntel") ? 1 : -1;

tools/perf/util/env.c

1078

struct perf_env env = { .total_mem = 0, };

tools/perf/util/env.c

108

bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)

tools/perf/util/env.c

1081

perf_env__init(&env);

tools/perf/util/env.c

1082

perf_env__cpuid(&env);

tools/perf/util/env.c

1083

is_intel = perf_env__is_x86_intel_cpu(&env);

tools/perf/util/env.c

1084

perf_env__exit(&env);

tools/perf/util/env.c

112

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

113

ret = __perf_env__insert_btf(env, btf_node);

tools/perf/util/env.c

114

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

118

bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)

tools/perf/util/env.c

125

p = &env->bpf_progs.btfs.rb_node;

tools/perf/util/env.c

141

rb_insert_color(&btf_node->rb_node, &env->bpf_progs.btfs);

tools/perf/util/env.c

142

env->bpf_progs.btfs_cnt++;

tools/perf/util/env.c

146

struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id)

tools/perf/util/env.c

150

down_read(&env->bpf_progs.lock);

tools/perf/util/env.c

151

res = __perf_env__find_btf(env, btf_id);

tools/perf/util/env.c

152

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

156

struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id)

tools/perf/util/env.c

161

n = env->bpf_progs.btfs.rb_node;

tools/perf/util/env.c

176

static void perf_env__purge_bpf(struct perf_env *env)

tools/perf/util/env.c

181

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

183

root = &env->bpf_progs.infos;

tools/perf/util/env.c

197

env->bpf_progs.infos_cnt = 0;

tools/perf/util/env.c

199

root = &env->bpf_progs.btfs;

tools/perf/util/env.c

211

env->bpf_progs.btfs_cnt = 0;

tools/perf/util/env.c

213

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

216

static void perf_env__purge_bpf(struct perf_env *env __maybe_unused)

tools/perf/util/env.c

249

void perf_env__exit(struct perf_env *env)

tools/perf/util/env.c

253

mutex_destroy(&env->lock);

tools/perf/util/env.c

255

perf_env__purge_bpf(env);

tools/perf/util/env.c

256

perf_env__purge_cgroups(env);

tools/perf/util/env.c

257

zfree(&env->hostname);

tools/perf/util/env.c

258

zfree(&env->os_release);

tools/perf/util/env.c

259

zfree(&env->version);

tools/perf/util/env.c

260

zfree(&env->arch);

tools/perf/util/env.c

261

zfree(&env->cpu_desc);

tools/perf/util/env.c

262

zfree(&env->cpuid);

tools/perf/util/env.c

263

zfree(&env->cmdline);

tools/perf/util/env.c

264

zfree(&env->cmdline_argv);

tools/perf/util/env.c

265

zfree(&env->sibling_dies);

tools/perf/util/env.c

266

zfree(&env->sibling_cores);

tools/perf/util/env.c

267

zfree(&env->sibling_threads);

tools/perf/util/env.c

268

zfree(&env->pmu_mappings);

tools/perf/util/env.c

269

zfree(&env->cpu);

tools/perf/util/env.c

270

for (i = 0; i < env->nr_cpu_pmu_caps; i++)

tools/perf/util/env.c

271

zfree(&env->cpu_pmu_caps[i]);

tools/perf/util/env.c

272

zfree(&env->cpu_pmu_caps);

tools/perf/util/env.c

273

zfree(&env->numa_map);

tools/perf/util/env.c

275

for (i = 0; i < env->nr_numa_nodes; i++)

tools/perf/util/env.c

276

perf_cpu_map__put(env->numa_nodes[i].map);

tools/perf/util/env.c

277

zfree(&env->numa_nodes);

tools/perf/util/env.c

279

for (i = 0; i < env->caches_cnt; i++)

tools/perf/util/env.c

280

cpu_cache_level__free(&env->caches[i]);

tools/perf/util/env.c

281

zfree(&env->caches);

tools/perf/util/env.c

283

for (i = 0; i < env->nr_memory_nodes; i++)

tools/perf/util/env.c

284

zfree(&env->memory_nodes[i].set);

tools/perf/util/env.c

285

zfree(&env->memory_nodes);

tools/perf/util/env.c

287

for (i = 0; i < env->nr_hybrid_nodes; i++) {

tools/perf/util/env.c

288

zfree(&env->hybrid_nodes[i].pmu_name);

tools/perf/util/env.c

289

zfree(&env->hybrid_nodes[i].cpus);

tools/perf/util/env.c

29

bool perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

291

zfree(&env->hybrid_nodes);

tools/perf/util/env.c

293

for (i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/env.c

294

for (j = 0; j < env->pmu_caps[i].nr_caps; j++)

tools/perf/util/env.c

295

zfree(&env->pmu_caps[i].caps[j]);

tools/perf/util/env.c

296

zfree(&env->pmu_caps[i].caps);

tools/perf/util/env.c

297

zfree(&env->pmu_caps[i].pmu_name);

tools/perf/util/env.c

299

zfree(&env->pmu_caps);

tools/perf/util/env.c

300

free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail);

tools/perf/util/env.c

303

void perf_env__init(struct perf_env *env)

tools/perf/util/env.c

305

memset(env, 0, sizeof(*env));

tools/perf/util/env.c

307

env->bpf_progs.infos = RB_ROOT;

tools/perf/util/env.c

308

env->bpf_progs.btfs = RB_ROOT;

tools/perf/util/env.c

309

init_rwsem(&env->bpf_progs.lock);

tools/perf/util/env.c

311

env->kernel_is_64_bit = -1;

tools/perf/util/env.c

312

mutex_init(&env->lock);

tools/perf/util/env.c

315

static void perf_env__init_kernel_mode(struct perf_env *env)

tools/perf/util/env.c

317

const char *arch = env->arch;

tools/perf/util/env.c

330

env->kernel_is_64_bit = 1;

tools/perf/util/env.c

332

env->kernel_is_64_bit = 0;

tools/perf/util/env.c

337

env->kernel_is_64_bit = (sizeof(void *) == 8) ? 1 : 0;

tools/perf/util/env.c

34

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

340

int perf_env__kernel_is_64_bit(struct perf_env *env)

tools/perf/util/env.c

342

if (env->kernel_is_64_bit == -1)

tools/perf/util/env.c

343

perf_env__init_kernel_mode(env);

tools/perf/util/env.c

345

return env->kernel_is_64_bit;

tools/perf/util/env.c

35

ret = __perf_env__insert_bpf_prog_info(env, info_node);

tools/perf/util/env.c

36

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

364

const char *perf_env__os_release(struct perf_env *env)

tools/perf/util/env.c

370

if (!env)

tools/perf/util/env.c

373

mutex_lock(&env->lock);

tools/perf/util/env.c

374

if (env->os_release) {

tools/perf/util/env.c

375

release = env->os_release;

tools/perf/util/env.c

384

if (env->arch) {

tools/perf/util/env.c

395

env->os_release = strdup(ret < 0 ? perf_version_string : uts.release);

tools/perf/util/env.c

396

release = env->os_release ?: perf_version_string;

tools/perf/util/env.c

398

mutex_unlock(&env->lock);

tools/perf/util/env.c

402

int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[])

tools/perf/util/env.c

407

env->cmdline_argv = calloc(argc, sizeof(char *));

tools/perf/util/env.c

408

if (env->cmdline_argv == NULL)

tools/perf/util/env.c

41

bool __perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node)

tools/perf/util/env.c

416

env->cmdline_argv[i] = argv[i];

tools/perf/util/env.c

417

if (env->cmdline_argv[i] == NULL)

tools/perf/util/env.c

421

env->nr_cmdline = argc;

tools/perf/util/env.c

425

zfree(&env->cmdline_argv);

tools/perf/util/env.c

430

int perf_env__read_cpu_topology_map(struct perf_env *env)

tools/perf/util/env.c

434

if (env->cpu != NULL)

tools/perf/util/env.c

437

if (env->nr_cpus_avail == 0)

tools/perf/util/env.c

438

env->nr_cpus_avail = cpu__max_present_cpu().cpu;

tools/perf/util/env.c

440

nr_cpus = env->nr_cpus_avail;

tools/perf/util/env.c

444

env->cpu = calloc(nr_cpus, sizeof(env->cpu[0]));

tools/perf/util/env.c

445

if (env->cpu == NULL)

tools/perf/util/env.c

454

env->cpu[idx].core_id = core_id >= 0 ? core_id : -1;

tools/perf/util/env.c

455

env->cpu[idx].socket_id = socket_id >= 0 ? socket_id : -1;

tools/perf/util/env.c

456

env->cpu[idx].die_id = die_id >= 0 ? die_id : -1;

tools/perf/util/env.c

459

env->nr_cpus_avail = nr_cpus;

tools/perf/util/env.c

463

int perf_env__read_pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

476

env->nr_pmu_mappings = pmu_num;

tools/perf/util/env.c

48

p = &env->bpf_progs.infos.rb_node;

tools/perf/util/env.c

489

env->pmu_mappings = strbuf_detach(&sb, NULL);

tools/perf/util/env.c

498

int perf_env__read_cpuid(struct perf_env *env)

tools/perf/util/env.c

507

free(env->cpuid);

tools/perf/util/env.c

508

env->cpuid = strdup(cpuid);

tools/perf/util/env.c

509

if (env->cpuid == NULL)

tools/perf/util/env.c

514

static int perf_env__read_nr_cpus_avail(struct perf_env *env)

tools/perf/util/env.c

516

if (env->nr_cpus_avail == 0)

tools/perf/util/env.c

517

env->nr_cpus_avail = cpu__max_present_cpu().cpu;

tools/perf/util/env.c

519

return env->nr_cpus_avail ? 0 : -ENOENT;

tools/perf/util/env.c

568

int perf_env__read_core_pmu_caps(struct perf_env *env)

tools/perf/util/env.c

587

return __perf_env__read_core_pmu_caps(pmu, &env->nr_cpu_pmu_caps,

tools/perf/util/env.c

588

&env->cpu_pmu_caps,

tools/perf/util/env.c

589

&env->max_branches,

tools/perf/util/env.c

590

&env->br_cntr_nr,

tools/perf/util/env.c

591

&env->br_cntr_width);

tools/perf/util/env.c

617

env->nr_pmus_with_caps = nr_pmu;

tools/perf/util/env.c

618

env->pmu_caps = pmu_caps;

tools/perf/util/env.c

632

int perf_env__nr_cpus_avail(struct perf_env *env)

tools/perf/util/env.c

634

return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0;

tools/perf/util/env.c

64

rb_insert_color(&info_node->rb_node, &env->bpf_progs.infos);

tools/perf/util/env.c

65

env->bpf_progs.infos_cnt++;

tools/perf/util/env.c

69

struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

75

down_read(&env->bpf_progs.lock);

tools/perf/util/env.c

76

n = env->bpf_progs.infos.rb_node;

tools/perf/util/env.c

782

uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags)

tools/perf/util/env.c

791

if (env) {

tools/perf/util/env.c

792

arch = env->arch;

tools/perf/util/env.c

793

is_64_bit = env->kernel_is_64_bit;

tools/perf/util/env.c

813

uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags)

tools/perf/util/env.c

818

if (env && env->e_machine != EM_NONE) {

tools/perf/util/env.c

820

*e_flags = env->e_flags;

tools/perf/util/env.c

822

return env->e_machine;

tools/perf/util/env.c

824

e_machine = perf_env__e_machine_nocache(env, &local_e_flags);

tools/perf/util/env.c

832

if (env && env->arch) {

tools/perf/util/env.c

833

env->e_machine = e_machine;

tools/perf/util/env.c

834

env->e_flags = local_e_flags;

tools/perf/util/env.c

842

const char *perf_env__arch(struct perf_env *env)

tools/perf/util/env.c

847

if (!env) {

tools/perf/util/env.c

864

e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);

tools/perf/util/env.c

867

if (e_machine == EM_RISCV && perf_env__kernel_is_64_bit(env) == 1)

tools/perf/util/env.c

869

else if (e_machine == EM_MIPS && perf_env__kernel_is_64_bit(env) == 1)

tools/perf/util/env.c

871

else if (e_machine == EM_PARISC && perf_env__kernel_is_64_bit(env) == 1)

tools/perf/util/env.c

884

const char *perf_env__cpuid(struct perf_env *env)

tools/perf/util/env.c

888

if (!env->cpuid) { /* Assume local operation */

tools/perf/util/env.c

889

status = perf_env__read_cpuid(env);

tools/perf/util/env.c

894

return env->cpuid;

tools/perf/util/env.c

897

int perf_env__nr_pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

90

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

901

if (!env->nr_pmu_mappings) { /* Assume local operation */

tools/perf/util/env.c

902

status = perf_env__read_pmu_mappings(env);

tools/perf/util/env.c

907

return env->nr_pmu_mappings;

tools/perf/util/env.c

910

const char *perf_env__pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

914

if (!env->pmu_mappings) { /* Assume local operation */

tools/perf/util/env.c

915

status = perf_env__read_pmu_mappings(env);

tools/perf/util/env.c

920

return env->pmu_mappings;

tools/perf/util/env.c

923

int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu)

tools/perf/util/env.c

925

if (!env->nr_numa_map) {

tools/perf/util/env.c

929

for (i = 0; i < env->nr_numa_nodes; i++) {

tools/perf/util/env.c

930

nn = &env->numa_nodes[i];

tools/perf/util/env.c

94

void perf_env__iterate_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

940

env->numa_map = malloc(nr * sizeof(int));

tools/perf/util/env.c

941

if (!env->numa_map)

tools/perf/util/env.c

945

env->numa_map[i] = -1;

tools/perf/util/env.c

947

env->nr_numa_map = nr;

tools/perf/util/env.c

949

for (i = 0; i < env->nr_numa_nodes; i++) {

tools/perf/util/env.c

953

nn = &env->numa_nodes[i];

tools/perf/util/env.c

955

env->numa_map[tmp.cpu] = i;

tools/perf/util/env.c

959

return cpu.cpu >= 0 && cpu.cpu < env->nr_numa_map ? env->numa_map[cpu.cpu] : -1;

tools/perf/util/env.c

962

bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name)

tools/perf/util/env.c

964

char *pmu_mapping = env->pmu_mappings, *colon;

tools/perf/util/env.c

966

for (int i = 0; i < env->nr_pmu_mappings; ++i) {

tools/perf/util/env.c

980

char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,

tools/perf/util/env.h

173

int perf_env__read_core_pmu_caps(struct perf_env *env);

tools/perf/util/env.h

175

void perf_env__exit(struct perf_env *env);

tools/perf/util/env.h

177

int perf_env__kernel_is_64_bit(struct perf_env *env);

tools/perf/util/env.h

179

const char *perf_env__os_release(struct perf_env *env);

tools/perf/util/env.h

181

int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]);

tools/perf/util/env.h

183

int perf_env__read_cpuid(struct perf_env *env);

tools/perf/util/env.h

184

int perf_env__read_pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

185

int perf_env__nr_pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

186

const char *perf_env__pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

188

int perf_env__read_cpu_topology_map(struct perf_env *env);

tools/perf/util/env.h

197

perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu)

tools/perf/util/env.h

199

if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail)

tools/perf/util/env.h

200

return &env->cpu[cpu.cpu];

tools/perf/util/env.h

206

uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags);

tools/perf/util/env.h

207

uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags);

tools/perf/util/env.h

208

const char *perf_env__arch(struct perf_env *env);

tools/perf/util/env.h

210

const char *perf_env__cpuid(struct perf_env *env);

tools/perf/util/env.h

211

int perf_env__nr_cpus_avail(struct perf_env *env);

tools/perf/util/env.h

213

void perf_env__init(struct perf_env *env);

tools/perf/util/env.h

215

bool __perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

217

bool perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

219

struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

221

void perf_env__iterate_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

225

bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);

tools/perf/util/env.h

226

bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);

tools/perf/util/env.h

227

struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id);

tools/perf/util/env.h

228

struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id);

tools/perf/util/env.h

231

int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu);

tools/perf/util/env.h

232

char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,

tools/perf/util/env.h

235

bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name);

tools/perf/util/env.h

236

void perf_env__find_br_cntr_info(struct perf_env *env,

tools/perf/util/env.h

241

bool perf_env__is_x86_amd_cpu(struct perf_env *env);

tools/perf/util/env.h

243

bool perf_env__is_x86_intel_cpu(struct perf_env *env);

tools/perf/util/event.c

838

struct perf_env *env = machine->env;

tools/perf/util/event.c

845

if (env && al->cpu < env->nr_cpus_avail) {

tools/perf/util/event.c

848

topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu });

tools/perf/util/header.c

1027

return do_write(ff, &ff->ph->env.clock.clockid_res_ns,

tools/perf/util/header.c

1028

sizeof(ff->ph->env.clock.clockid_res_ns));

tools/perf/util/header.c

1046

data32 = ff->ph->env.clock.clockid;

tools/perf/util/header.c

1053

data64 = &ff->ph->env.clock.tod_ns;

tools/perf/util/header.c

1060

data64 = &ff->ph->env.clock.clockid_ns;

tools/perf/util/header.c

1118

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

1123

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1125

ret = do_write(ff, &env->bpf_progs.infos_cnt,

tools/perf/util/header.c

1126

sizeof(env->bpf_progs.infos_cnt));

tools/perf/util/header.c

1127

if (ret < 0 || env->bpf_progs.infos_cnt == 0)

tools/perf/util/header.c

1130

root = &env->bpf_progs.infos;

tools/perf/util/header.c

1153

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1165

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

1170

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1172

ret = do_write(ff, &env->bpf_progs.btfs_cnt,

tools/perf/util/header.c

1173

sizeof(env->bpf_progs.btfs_cnt));

tools/perf/util/header.c

1175

if (ret < 0 || env->bpf_progs.btfs_cnt == 0)

tools/perf/util/header.c

1178

root = &env->bpf_progs.btfs;

tools/perf/util/header.c

1191

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1408

ff->ph->env.cln_size = cln_size;

tools/perf/util/header.c

1628

ret = do_write(ff, &(ff->ph->env.comp_ver), sizeof(ff->ph->env.comp_ver));

tools/perf/util/header.c

1632

ret = do_write(ff, &(ff->ph->env.comp_type), sizeof(ff->ph->env.comp_type));

tools/perf/util/header.c

1636

ret = do_write(ff, &(ff->ph->env.comp_level), sizeof(ff->ph->env.comp_level));

tools/perf/util/header.c

1640

ret = do_write(ff, &(ff->ph->env.comp_ratio), sizeof(ff->ph->env.comp_ratio));

tools/perf/util/header.c

1644

return do_write(ff, &(ff->ph->env.comp_mmap_len), sizeof(ff->ph->env.comp_mmap_len));

tools/perf/util/header.c

1896

fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname);

tools/perf/util/header.c

1901

fprintf(fp, "# os release : %s\n", ff->ph->env.os_release);

tools/perf/util/header.c

1906

fprintf(fp, "# arch : %s\n", ff->ph->env.arch);

tools/perf/util/header.c

1911

fprintf(fp, "# e_machine : %u\n", ff->ph->env.e_machine);

tools/perf/util/header.c

1912

fprintf(fp, "# e_flags : %u\n", ff->ph->env.e_flags);

tools/perf/util/header.c

1917

fprintf(fp, "# cpudesc : %s\n", ff->ph->env.cpu_desc);

tools/perf/util/header.c

1922

fprintf(fp, "# nrcpus online : %u\n", ff->ph->env.nr_cpus_online);

tools/perf/util/header.c

1923

fprintf(fp, "# nrcpus avail : %u\n", ff->ph->env.nr_cpus_avail);

tools/perf/util/header.c

1928

fprintf(fp, "# perf version : %s\n", ff->ph->env.version);

tools/perf/util/header.c

1935

nr = ff->ph->env.nr_cmdline;

tools/perf/util/header.c

1940

char *argv_i = strdup(ff->ph->env.cmdline_argv[i]);

tools/perf/util/header.c

1942

fprintf(fp, "%s ", ff->ph->env.cmdline_argv[i]);

tools/perf/util/header.c

1963

int cpu_nr = ph->env.nr_cpus_avail;

tools/perf/util/header.c

1967

nr = ph->env.nr_sibling_cores;

tools/perf/util/header.c

1968

str = ph->env.sibling_cores;

tools/perf/util/header.c

1975

if (ph->env.nr_sibling_dies) {

tools/perf/util/header.c

1976

nr = ph->env.nr_sibling_dies;

tools/perf/util/header.c

1977

str = ph->env.sibling_dies;

tools/perf/util/header.c

1985

nr = ph->env.nr_sibling_threads;

tools/perf/util/header.c

1986

str = ph->env.sibling_threads;

tools/perf/util/header.c

1993

if (ph->env.nr_sibling_dies) {

tools/perf/util/header.c

1994

if (ph->env.cpu != NULL) {

tools/perf/util/header.c

1998

i, ph->env.cpu[i].core_id,

tools/perf/util/header.c

1999

ph->env.cpu[i].die_id,

tools/perf/util/header.c

2000

ph->env.cpu[i].socket_id);

tools/perf/util/header.c

2005

if (ph->env.cpu != NULL) {

tools/perf/util/header.c

2009

i, ph->env.cpu[i].core_id,

tools/perf/util/header.c

2010

ph->env.cpu[i].socket_id);

tools/perf/util/header.c

2020

ff->ph->env.clock.clockid_res_ns * 1000);

tools/perf/util/header.c

2032

if (!ff->ph->env.clock.enabled) {

tools/perf/util/header.c

2038

ref = ff->ph->env.clock.tod_ns;

tools/perf/util/header.c

2044

ref = ff->ph->env.clock.clockid_ns;

tools/perf/util/header.c

2049

clockid = ff->ph->env.clock.clockid;

tools/perf/util/header.c

2072

for (i = 0; i < ff->ph->env.nr_hybrid_nodes; i++) {

tools/perf/util/header.c

2073

n = &ff->ph->env.hybrid_nodes[i];

tools/perf/util/header.c

2092

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2096

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2098

root = &env->bpf_progs.infos;

tools/perf/util/header.c

2110

__bpf_event__print_bpf_prog_info(node->info_linear, env, fp);

tools/perf/util/header.c

2113

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2122

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2126

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2128

root = &env->bpf_progs.btfs;

tools/perf/util/header.c

2142

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2378

fprintf(fp, "# total memory : %llu kB\n", ff->ph->env.total_mem);

tools/perf/util/header.c

2386

for (i = 0; i < ff->ph->env.nr_numa_nodes; i++) {

tools/perf/util/header.c

2387

n = &ff->ph->env.numa_nodes[i];

tools/perf/util/header.c

2400

fprintf(fp, "# cpuid : %s\n", ff->ph->env.cpuid);

tools/perf/util/header.c

2423

for (i = 0; i < ff->ph->env.caches_cnt; i++) {

tools/perf/util/header.c

2425

cpu_cache_level__fprintf(fp, &ff->ph->env.caches[i]);

tools/perf/util/header.c

2431

fprintf(fp, "# cacheline size: %u\n", ff->ph->env.cln_size);

tools/perf/util/header.c

2437

ff->ph->env.comp_type == PERF_COMP_ZSTD ? "Zstd" : "Unknown",

tools/perf/util/header.c

2438

ff->ph->env.comp_level, ff->ph->env.comp_ratio);

tools/perf/util/header.c

2462

__print_pmu_caps(fp, ff->ph->env.nr_cpu_pmu_caps,

tools/perf/util/header.c

2463

ff->ph->env.cpu_pmu_caps, (char *)"cpu");

tools/perf/util/header.c

2468

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2469

uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);

tools/perf/util/header.c

2471

for (int i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/header.c

2472

struct pmu_caps *pmu_caps = &env->pmu_caps[i];

tools/perf/util/header.c

2479

perf_env__has_pmu_mapping(env, "ibs_op")) {

tools/perf/util/header.c

2480

char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise");

tools/perf/util/header.c

2489

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2495

pmu_num = env->nr_pmu_mappings;

tools/perf/util/header.c

2501

str = env->pmu_mappings;

tools/perf/util/header.c

2583

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2587

nodes = env->memory_nodes;

tools/perf/util/header.c

2588

nr = env->nr_memory_nodes;

tools/perf/util/header.c

2591

nr, env->memory_bsize);

tools/perf/util/header.c

2594

memory_node__fprintf(&nodes[i], env->memory_bsize, fp);

tools/perf/util/header.c

2600

struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain;

tools/perf/util/header.c

2601

u32 nr = ff->ph->env.nr_cpus_avail;

tools/perf/util/header.c

2605

fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version);

tools/perf/util/header.c

2606

fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains);

tools/perf/util/header.c

2622

if (ff->ph->env.schedstat_version >= 17)

tools/perf/util/header.c

2846

free(ff->ph->env.__feat_env); \

tools/perf/util/header.c

2847

ff->ph->env.__feat_env = do_read_string(ff); \

tools/perf/util/header.c

2848

return ff->ph->env.__feat_env ? 0 : -ENOMEM; \

tools/perf/util/header.c

2859

free(ff->ph->env.arch);

tools/perf/util/header.c

2860

ff->ph->env.arch = do_read_string(ff);

tools/perf/util/header.c

2861

if (!ff->ph->env.arch)

tools/perf/util/header.c

2870

ret = do_read_u32(ff, &ff->ph->env.e_machine);

tools/perf/util/header.c

2874

return do_read_u32(ff, &ff->ph->env.e_flags);

tools/perf/util/header.c

2905

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2934

env->nr_cpus_avail = (int)nr_cpus_avail;

tools/perf/util/header.c

2935

env->nr_cpus_online = (int)nr_cpus_online;

tools/perf/util/header.c

2941

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2948

env->total_mem = (unsigned long long)total_mem;

tools/perf/util/header.c

3015

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3025

env->nr_cmdline = nr;

tools/perf/util/header.c

3045

env->cmdline = cmdline;

tools/perf/util/header.c

3046

env->cmdline_argv = (const char **) argv;

tools/perf/util/header.c

3060

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3061

int cpu_nr = env->nr_cpus_avail;

tools/perf/util/header.c

3069

env->cpu = calloc(cpu_nr, sizeof(*env->cpu));

tools/perf/util/header.c

3070

if (!env->cpu)

tools/perf/util/header.c

3082

env->nr_sibling_cores = nr;

tools/perf/util/header.c

3098

env->sibling_cores = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

3109

env->nr_sibling_threads = nr;

tools/perf/util/header.c

3123

env->sibling_threads = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

3130

zfree(&env->cpu);

tools/perf/util/header.c

3138

env->cpu[i].core_id = nr;

tools/perf/util/header.c

3144

env->cpu[i].socket_id = nr;

tools/perf/util/header.c

3164

env->nr_sibling_dies = nr;

tools/perf/util/header.c

3178

env->sibling_dies = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

3184

env->cpu[i].die_id = nr;

tools/perf/util/header.c

3193

zfree(&env->cpu);

tools/perf/util/header.c

3199

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3246

env->nr_numa_nodes = nr;

tools/perf/util/header.c

3247

env->numa_nodes = nodes;

tools/perf/util/header.c

3257

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3283

env->nr_pmu_mappings = pmu_num;

tools/perf/util/header.c

3302

env->msr_pmu_type = type;

tools/perf/util/header.c

3308

free(env->pmu_mappings);

tools/perf/util/header.c

3309

env->pmu_mappings = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

3319

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3350

env->nr_groups = nr_groups;

tools/perf/util/header.c

3430

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3483

env->caches = caches;

tools/perf/util/header.c

3484

env->caches_cnt = cnt;

tools/perf/util/header.c

3498

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3500

if (do_read_u32(ff, &env->cln_size))

tools/perf/util/header.c

3530

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3582

env->memory_bsize = bsize;

tools/perf/util/header.c

3583

env->memory_nodes = nodes;

tools/perf/util/header.c

3584

env->nr_memory_nodes = nr;

tools/perf/util/header.c

3596

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3598

if (do_read_u64(ff, &env->clock.clockid_res_ns))

tools/perf/util/header.c

3607

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3622

env->clock.clockid = data32;

tools/perf/util/header.c

3628

env->clock.tod_ns = data64;

tools/perf/util/header.c

3634

env->clock.clockid_ns = data64;

tools/perf/util/header.c

3635

env->clock.enabled = true;

tools/perf/util/header.c

3642

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3678

env->nr_hybrid_nodes = nr;

tools/perf/util/header.c

3679

env->hybrid_nodes = nodes;

tools/perf/util/header.c

3711

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3736

down_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3784

if (!__perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/header.c

3790

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3795

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3807

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3831

down_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3857

if (!__perf_env__insert_btf(env, node))

tools/perf/util/header.c

3864

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3877

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3879

if (do_read_u32(ff, &(env->comp_ver)))

tools/perf/util/header.c

3882

if (do_read_u32(ff, &(env->comp_type)))

tools/perf/util/header.c

3885

if (do_read_u32(ff, &(env->comp_level)))

tools/perf/util/header.c

3888

if (do_read_u32(ff, &(env->comp_ratio)))

tools/perf/util/header.c

3891

if (do_read_u32(ff, &(env->comp_mmap_len)))

tools/perf/util/header.c

3905

if (env->comp_mmap_len < 4096 || env->comp_mmap_len % 4096) {

tools/perf/util/header.c

3907

env->comp_mmap_len);

tools/perf/util/header.c

3986

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3987

int ret = __process_pmu_caps(ff, &env->nr_cpu_pmu_caps,

tools/perf/util/header.c

3988

&env->cpu_pmu_caps,

tools/perf/util/header.c

3989

&env->max_branches,

tools/perf/util/header.c

3990

&env->br_cntr_nr,

tools/perf/util/header.c

3991

&env->br_cntr_width);

tools/perf/util/header.c

3993

if (!ret && !env->cpu_pmu_caps)

tools/perf/util/header.c

4000

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

4050

env->nr_pmus_with_caps = nr_pmu;

tools/perf/util/header.c

4051

env->pmu_caps = pmu_caps;

tools/perf/util/header.c

4069

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

4076

nra = env->nr_cpus_avail;

tools/perf/util/header.c

4077

nr = env->nr_cpus_online;

tools/perf/util/header.c

4094

env->cpu_domain = cd_map;

tools/perf/util/header.c

4100

env->schedstat_version = schedstat_version;

tools/perf/util/header.c

4118

env->max_sched_domains = max_sched_domains;

tools/perf/util/header.c

5114

session->machines.host.env = &header->env;

tools/perf/util/header.c

5295

if (header->env.nr_cpus_avail == 0) {

tools/perf/util/header.c

5296

header->env.nr_cpus_avail = MAX_NR_CPUS;

tools/perf/util/header.c

5357

feat, header->env.version, perf_version_string);

tools/perf/util/header.c

660

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

668

n = env->nr_cmdline + 1;

tools/perf/util/header.c

678

for (i = 0 ; i < env->nr_cmdline; i++) {

tools/perf/util/header.c

679

ret = do_write_string(ff, env->cmdline_argv[i]);

tools/perf/util/header.c

690

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

718

ret = perf_env__read_cpu_topology_map(env);

tools/perf/util/header.c

722

for (j = 0; j < env->nr_cpus_avail; j++) {

tools/perf/util/header.c

723

ret = do_write(ff, &env->cpu[j].core_id,

tools/perf/util/header.c

724

sizeof(env->cpu[j].core_id));

tools/perf/util/header.c

727

ret = do_write(ff, &env->cpu[j].socket_id,

tools/perf/util/header.c

728

sizeof(env->cpu[j].socket_id));

tools/perf/util/header.c

746

for (j = 0; j < env->nr_cpus_avail; j++) {

tools/perf/util/header.c

747

ret = do_write(ff, &env->cpu[j].die_id,

tools/perf/util/header.c

748

sizeof(env->cpu[j].die_id));

tools/perf/util/header.h

114

struct perf_env env;

tools/perf/util/hist.c

257

struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(h->ms.thread))->env,

tools/perf/util/hist.h

728

float min_pcnt, struct perf_env *env, bool warn_lost_event);

tools/perf/util/hist.h

738

float min_percent, struct perf_env *env);

tools/perf/util/hist.h

745

struct perf_env *env __maybe_unused,

tools/perf/util/hist.h

786

struct perf_env *env __maybe_unused)

tools/perf/util/intel-pt.c

1297

struct perf_env *env = pt->machine->env;

tools/perf/util/intel-pt.c

1384

if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))

tools/perf/util/intel-pt.c

3591

struct perf_env *env = pt->machine->env;

tools/perf/util/intel-pt.c

3594

if (!env->cpuid)

tools/perf/util/intel-pt.c

3605

if (!strncmp(env->cpuid, "GenuineIntel,6,190,", 19))

tools/perf/util/intel-pt.c

3608

if (!strncmp(env->cpuid, "GenuineIntel,6,175,", 19) ||

tools/perf/util/intel-pt.c

3609

!strncmp(env->cpuid, "GenuineIntel,6,182,", 19))

tools/perf/util/libbfd.c

546

info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env,

tools/perf/util/libbfd.c

564

node = perf_env__find_btf(dso__bpf_prog(dso)->env,

tools/perf/util/lock-contention.c

107

uint16_t e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL);

tools/perf/util/machine.c

149

machine->env = host_env;

tools/perf/util/machine.c

1648

if (!machine->env) {

tools/perf/util/machine.c

1654

return perf_env__e_machine(machine->env, NULL) == e_machine;

tools/perf/util/machine.c

2845

struct perf_env *env = evsel__env(evsel);

tools/perf/util/machine.c

2849

!env ? 0 : env->max_branches,

tools/perf/util/machine.c

3189

return machine ? perf_env__nr_cpus_avail(machine->env) : 0;

tools/perf/util/machine.c

673

cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);

tools/perf/util/machine.c

91

machine->env = NULL;

tools/perf/util/machine.h

44

struct perf_env *env;

tools/perf/util/mem2node.c

47

int mem2node__init(struct mem2node *map, struct perf_env *env)

tools/perf/util/mem2node.c

49

struct memory_node *n, *nodes = &env->memory_nodes[0];

tools/perf/util/mem2node.c

51

u64 bsize = env->memory_bsize;

tools/perf/util/mem2node.c

57

for (i = 0; i < env->nr_memory_nodes; i++) {

tools/perf/util/mem2node.c

66

for (i = 0; i < env->nr_memory_nodes; i++) {

tools/perf/util/mem2node.h

16

int mem2node__init(struct mem2node *map, struct perf_env *env);

tools/perf/util/parse-events.c

583

struct perf_env env = {};

tools/perf/util/parse-events.c

585

perf_env__init(&env);

tools/perf/util/parse-events.c

586

len = perf_env__kernel_is_64_bit(&env) ? sizeof(u64) : sizeof(long);

tools/perf/util/parse-events.c

587

perf_env__exit(&env);

tools/perf/util/print_insn.c

46

e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL);

tools/perf/util/sample-raw.c

16

void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sample-raw.c

18

uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);

tools/perf/util/sample-raw.c

23

const char *cpuid = perf_env__cpuid(env);

tools/perf/util/sample-raw.h

17

void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env);

tools/perf/util/session.c

163

perf_env__init(&session->header.env);

tools/perf/util/session.c

190

evlist__init_trace_event_sample_raw(session->evlist, &session->header.env);

tools/perf/util/session.c

205

session->machines.host.env = host_env;

tools/perf/util/session.c

211

perf_env__single_address_space(session->machines.host.env);

tools/perf/util/session.c

264

perf_env__exit(&session->header.env);

tools/perf/util/session.c

4062

return &session->header.env;

tools/perf/util/session.c

4087

struct perf_env *env;

tools/perf/util/session.c

4101

env = perf_session__env(session);

tools/perf/util/session.c

4102

if (env && env->e_machine != EM_NONE)

tools/perf/util/session.c

4103

return perf_env__e_machine(env, e_flags);

tools/perf/util/session.c

4115

if (env) {

tools/perf/util/session.c

4116

env->e_machine = args.e_machine;

tools/perf/util/session.c

4117

env->e_flags = args.e_flags;

tools/perf/util/session.c

4131

return perf_env__e_machine_nocache(env, e_flags);

tools/perf/util/sort.c

1144

struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(he->ms.thread))->env,

tools/perf/util/sort.c

2613

ret = session->header.env.cln_size;

tools/perf/util/sort.c

2679

static int arch_support_sort_key(const char *sort_key, struct perf_env *env)

tools/perf/util/sort.c

2681

uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL);

tools/perf/util/sort.c

2693

static const char *arch_perf_header_entry(const char *se_header, struct perf_env *env)

tools/perf/util/sort.c

2695

uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL);

tools/perf/util/sort.c

2715

static void sort_dimension_add_dynamic_header(struct sort_dimension *sd, struct perf_env *env)

tools/perf/util/sort.c

2717

sd->entry->se_header = arch_perf_header_entry(sd->entry->se_header, env);

tools/perf/util/sort.c

3825

struct evlist *evlist, struct perf_env *env,

tools/perf/util/sort.c

3838

!arch_support_sort_key(tok, env)) {

tools/perf/util/sort.c

3851

sort_dimension_add_dynamic_header(sd, env);

tools/perf/util/sort.c

3905

static bool is_hpp_sort_key(const char *key, struct perf_env *env)

tools/perf/util/sort.c

3911

!arch_support_sort_key(key, env)) {

tools/perf/util/sort.c

3933

struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

3962

if (is_hpp_sort_key(tok, env)) {

tools/perf/util/sort.c

3975

ret = sort_dimension__add(list, tok, evlist, env, level);

tools/perf/util/sort.c

4104

static int __setup_sorting(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

4144

ret = setup_sort_list(&perf_hpp_list, str, evlist, env);

tools/perf/util/sort.c

4381

int setup_sorting(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

4385

err = __setup_sorting(evlist, env);

tools/perf/util/sort.c

4390

err = sort_dimension__add(&perf_hpp_list, "parent", evlist, env, -1);

tools/perf/util/sort.h

135

int setup_sorting(struct evlist *evlist, struct perf_env *env);

tools/perf/util/sort.h

150

struct evlist *evlist, struct perf_env *env,

tools/perf/util/svghelper.c

753

int svg_build_topology_map(struct perf_env *env)

tools/perf/util/svghelper.c

760

nr_cpus = min(env->nr_cpus_online, MAX_NR_CPUS);

tools/perf/util/svghelper.c

762

t.sib_core_nr = env->nr_sibling_cores;

tools/perf/util/svghelper.c

763

t.sib_thr_nr = env->nr_sibling_threads;

tools/perf/util/svghelper.c

764

t.sib_core = calloc(env->nr_sibling_cores, sizeof(cpumask_t));

tools/perf/util/svghelper.c

765

t.sib_thr = calloc(env->nr_sibling_threads, sizeof(cpumask_t));

tools/perf/util/svghelper.c

767

sib_core = env->sibling_cores;

tools/perf/util/svghelper.c

768

sib_thr = env->sibling_threads;

tools/perf/util/svghelper.c

775

for (i = 0; i < env->nr_sibling_cores; i++) {

tools/perf/util/svghelper.c

784

for (i = 0; i < env->nr_sibling_threads; i++) {

tools/perf/util/svghelper.h

33

int svg_build_topology_map(struct perf_env *env);

tools/perf/util/symbol.c

2336

static int vmlinux_path__init(struct perf_env *env)

tools/perf/util/symbol.c

2356

if (env) {

tools/perf/util/symbol.c

2357

kernel_version = perf_env__os_release(env);

tools/perf/util/symbol.c

2522

int symbol__init(struct perf_env *env)

tools/perf/util/symbol.c

2533

if (symbol_conf.try_vmlinux_path && vmlinux_path__init(env) < 0)

tools/perf/util/symbol.c

781

bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env)

tools/perf/util/symbol.c

826

e_machine = (env && env->arch) ? perf_env__e_machine(env, NULL) : EM_NONE;

tools/perf/util/symbol.c

829

if (e_machine == EM_NONE && env)

tools/perf/util/symbol.c

830

e_machine = perf_env__e_machine(env, NULL);

tools/perf/util/symbol.c

850

const char *release = env ? perf_env__os_release(env) : NULL;

tools/perf/util/symbol.c

975

if (machine && machine->env && machine->env->e_machine != EM_NONE)

tools/perf/util/symbol.c

976

return machine->env->e_machine;

tools/perf/util/symbol.c

978

return perf_env__e_machine(machine ? machine->env : NULL, /*e_flags=*/NULL);

tools/perf/util/symbol.h

136

bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env);

tools/perf/util/symbol.h

220

int symbol__init(struct perf_env *env);

tools/perf/util/thread.c

485

machine && machine->env ? perf_env__arch(machine->env) : NULL);

tools/perf/util/thread.c

487

return perf_env__e_machine(machine ? machine->env : NULL, e_flags);

tools/perf/util/thread.c

545

} else if (machine && machine->env) {

tools/perf/util/thread.c

547

e_machine = perf_env__e_machine(machine->env, &local_e_flags);

tools/perf/util/thread.c

548

args.is_big_endian = perf_arch_is_big_endian(perf_env__arch(machine->env));

tools/testing/selftests/bpf/bench.c

152

hits_mean, hits_stddev, hits_mean / env.producer_cnt);

tools/testing/selftests/bpf/bench.c

16

struct env env = {

tools/testing/selftests/bpf/bench.c

164

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/bench.c

188

hits_mean, hits_stddev, hits_mean / env.producer_cnt);

tools/testing/selftests/bpf/bench.c

189

printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);

tools/testing/selftests/bpf/bench.c

28

if (level == LIBBPF_DEBUG && !env.verbose)

tools/testing/selftests/bpf/bench.c

317

env.verbose = true;

tools/testing/selftests/bpf/bench.c

320

env.list = true;

tools/testing/selftests/bpf/bench.c

323

env.duration_sec = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

324

if (env.duration_sec <= 0) {

tools/testing/selftests/bpf/bench.c

330

env.warmup_sec = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

331

if (env.warmup_sec <= 0) {

tools/testing/selftests/bpf/bench.c

337

env.producer_cnt = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

338

if (env.producer_cnt < 0) {

tools/testing/selftests/bpf/bench.c

344

env.consumer_cnt = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

345

if (env.consumer_cnt < 0) {

tools/testing/selftests/bpf/bench.c

351

env.affinity = true;

tools/testing/selftests/bpf/bench.c

354

env.quiet = true;

tools/testing/selftests/bpf/bench.c

357

env.stacktrace = true;

tools/testing/selftests/bpf/bench.c

360

env.affinity = true;

tools/testing/selftests/bpf/bench.c

361

if (parse_num_list(arg, &env.prod_cpus.cpus,

tools/testing/selftests/bpf/bench.c

362

&env.prod_cpus.cpus_len)) {

tools/testing/selftests/bpf/bench.c

368

env.affinity = true;

tools/testing/selftests/bpf/bench.c

369

if (parse_num_list(arg, &env.cons_cpus.cpus,

tools/testing/selftests/bpf/bench.c

370

&env.cons_cpus.cpus_len)) {

tools/testing/selftests/bpf/bench.c

381

env.bench_name = strdup(arg);

tools/testing/selftests/bpf/bench.c

489

return cpu_set->next_cpu++ % env.nr_cpus;

tools/testing/selftests/bpf/bench.c

674

if (!env.bench_name) {

tools/testing/selftests/bpf/bench.c

679

if (strcmp(benchs[i]->name, env.bench_name) == 0) {

tools/testing/selftests/bpf/bench.c

685

fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);

tools/testing/selftests/bpf/bench.c

694

if (!env.quiet)

tools/testing/selftests/bpf/bench.c

697

state.producers = calloc(env.producer_cnt, sizeof(*state.producers));

tools/testing/selftests/bpf/bench.c

698

state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));

tools/testing/selftests/bpf/bench.c

699

state.results = calloc(env.duration_sec + env.warmup_sec + 2,

tools/testing/selftests/bpf/bench.c

709

for (i = 0; i < env.consumer_cnt; i++) {

tools/testing/selftests/bpf/bench.c

721

if (env.affinity)

tools/testing/selftests/bpf/bench.c

723

next_cpu(&env.cons_cpus));

tools/testing/selftests/bpf/bench.c

729

if (!env.prod_cpus.cpus)

tools/testing/selftests/bpf/bench.c

730

env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;

tools/testing/selftests/bpf/bench.c

732

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/bench.c

74

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/bench.c

744

if (env.affinity)

tools/testing/selftests/bpf/bench.c

746

next_cpu(&env.prod_cpus));

tools/testing/selftests/bpf/bench.c

749

if (!env.quiet)

tools/testing/selftests/bpf/bench.c

772

if (iter == env.duration_sec + env.warmup_sec)

tools/testing/selftests/bpf/bench.c

778

env.nr_cpus = get_nprocs();

tools/testing/selftests/bpf/bench.c

781

if (env.list) {

tools/testing/selftests/bpf/bench.c

804

bench->report_final(state.results + env.warmup_sec,

tools/testing/selftests/bpf/bench.c

805

state.res_cnt - env.warmup_sec);

tools/testing/selftests/bpf/bench.h

69

extern struct env env;

tools/testing/selftests/bpf/bench_bpf_timing.h

40

(t)->nr_cpus = env.nr_cpus; \

tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c

110

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_crypto.c

71

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c

17

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

116

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

269

if (env.quiet) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

271

if (env.affinity)

tools/testing/selftests/bpf/benchs/bench_bpf_loop.c

50

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_nop.c

17

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_nop.c

56

env.duration_sec = 600;

tools/testing/selftests/bpf/benchs/bench_bpf_timing.c

129

if (t->warmup_ticks < env.warmup_sec)

tools/testing/selftests/bpf/benchs/bench_bpf_timing.c

132

if (t->warmup_ticks == env.warmup_sec) {

tools/testing/selftests/bpf/benchs/bench_bpf_timing.c

150

if (total_samples >= (__u32)env.producer_cnt * t->target_samples && !t->done) {

tools/testing/selftests/bpf/benchs/bench_count.c

38

ctx->hits = calloc(env.producer_cnt, sizeof(*ctx->hits));

tools/testing/selftests/bpf/benchs/bench_count.c

59

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

112

nr = (env.producer_cnt + 1) / 2;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

132

nr = (env.producer_cnt + 1) / 2;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

184

bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus));

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

200

ctx.skel->bss->nr_thread = env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

305

res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

98

if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) {

tools/testing/selftests/bpf/benchs/bench_local_storage.c

73

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_local_storage.c

77

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

104

threads = calloc(env.producer_cnt, sizeof(*threads));

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

111

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

207

creates_per_sec, creates_per_sec / env.producer_cnt);

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

229

creates_mean, creates_stddev, creates_mean / env.producer_cnt,

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

74

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

215

if (env.quiet)

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

71

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

75

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

107

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

122

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

137

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

372

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

407

hits_mean, hits_stddev, rate, hits_mean / env.producer_cnt,

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

410

latency / lat_divisor / env.producer_cnt, unit);

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

87

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_rename.c

16

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_rename.c

20

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

112

if (args.bench_producer && strcmp(env.bench_name, "rb-libbpf")) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

122

if (args.bench_producer && env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

137

if (!args.bench_producer && env.consumer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

142

if (args.back2back && env.producer_cnt > 1) {

tools/testing/selftests/bpf/benchs/bench_sockmap.c

238

if (env.consumer_cnt != 2 || env.producer_cnt != 1 ||

tools/testing/selftests/bpf/benchs/bench_sockmap.c

239

!env.affinity)

tools/testing/selftests/bpf/benchs/bench_strncmp.c

53

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_trigger.c

149

ctx.skel->rodata->stacktrace = env.stacktrace;

tools/testing/selftests/bpf/benchs/bench_trigger.c

97

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_xdp_lb.c

1023

env.duration_sec = 600;

tools/testing/selftests/bpf/benchs/bench_xdp_lb.c

1102

env.quiet = true;

tools/testing/selftests/bpf/benchs/bench_xdp_lb.c

956

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/jit_disasm_helpers.c

249

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c

109

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c

201

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c

123

!ASSERT_FALSE((env.jit_enabled && !prog_infos[i].jited_prog_len), "jited_prog_len") ||

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c

124

!ASSERT_FALSE((env.jit_enabled && !memcmp(jited_insns, zeros, sizeof(zeros))),

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c

55

if (env.verifier_stats) {

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c

66

if (env.verifier_stats)

tools/testing/selftests/bpf/prog_tests/btf.c

8336

always_log = env.verbosity > VERBOSE_NONE;

tools/testing/selftests/bpf/prog_tests/btf_module.c

15

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/btf_tag.c

60

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/build_id.c

61

if (env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/build_id.c

97

if (env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/core_reloc.c

1026

if (test_case->needs_testmod && !env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c

105

if (prog_fd < 0 || env.verbosity > VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c

77

if (btf_fd < 0 || env.verbosity > VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c

393

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c

398

if (env.verbosity > VERBOSE_NONE && exp_msg) /* we overtook log */

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c

47

if (env.jit_enabled) {

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c

64

if (env.jit_enabled) {

tools/testing/selftests/bpf/prog_tests/iters.c

305

if (env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/iters.c

75

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/ksyms_module.c

19

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/ksyms_module.c

46

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/log_buf.c

132

if (env.verbosity > VERBOSE_NONE) {

tools/testing/selftests/bpf/prog_tests/log_buf.c

91

if (env.verbosity > VERBOSE_NONE) {

tools/testing/selftests/bpf/prog_tests/log_fixup.c

131

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

160

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

66

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

97

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/module_fentry_shadow.c

65

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/nested_trust.c

14

if (env.has_testmod)

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

15

struct test_state *state = env.test_state;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

166

error_cnt = env.subtest_state->error_cnt;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

173

env.subtest_state->error_cnt = error_cnt;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

175

ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, cases[i].expected, "expected output");

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

177

ASSERT_STREQ(env.subtest_state->log_buf, "", "expected no output");

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1437

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1449

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1466

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1590

if (failed || env.verbosity >= VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1591

if (failed || env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1635

if (env.verbosity >= VERBOSE_NORMAL) /* this speeds up debugging */

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1639

if (err || env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1653

fprintf(env.stderr_saved, "PROGRESS (%s): %d/%d (%.2lf%%), "

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1734

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1766

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1786

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1819

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

812

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

824

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/sk_assign.c

69

(env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose");

tools/testing/selftests/bpf/prog_tests/summarization.c

29

if (env.verbosity >= VERBOSE_SUPER)

tools/testing/selftests/bpf/prog_tests/summarization.c

9

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c

8

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

715

char *env;

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

717

env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC");

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

718

if (env)

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

719

return atoi(env);

tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c

92

if (env.verbosity >= VERBOSE_NORMAL) {

tools/testing/selftests/bpf/test_loader.c

1134

if (env.verbosity != VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

750

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

752

if (env.verbosity > VERBOSE_VERY)

tools/testing/selftests/bpf/test_loader.c

774

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

781

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

788

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

795

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

839

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

980

if (env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_progs.c

1030

struct test_env *env = state->input;

tools/testing/selftests/bpf/test_progs.c

1040

&env->subtest_selector.num_set,

tools/testing/selftests/bpf/test_progs.c

1041

&env->subtest_selector.num_set_len)) {

tools/testing/selftests/bpf/test_progs.c

1047

if (parse_num_list(arg, &env->test_selector.num_set,

tools/testing/selftests/bpf/test_progs.c

1048

&env->test_selector.num_set_len)) {

tools/testing/selftests/bpf/test_progs.c

1058

&env->test_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

1062

&env->test_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

107

if (env.subtest_state) {

tools/testing/selftests/bpf/test_progs.c

1071

&env->test_selector.blacklist,

tools/testing/selftests/bpf/test_progs.c

1075

&env->test_selector.blacklist,

tools/testing/selftests/bpf/test_progs.c

108

if (env.subtest_state->stdout_saved)

tools/testing/selftests/bpf/test_progs.c

1081

env->verifier_stats = true;

tools/testing/selftests/bpf/test_progs.c

1084

env->verbosity = VERBOSE_NORMAL;

tools/testing/selftests/bpf/test_progs.c

1087

env->verbosity = VERBOSE_VERY;

tools/testing/selftests/bpf/test_progs.c

109

fclose(env.subtest_state->stdout_saved);

tools/testing/selftests/bpf/test_progs.c

1090

env->verbosity = VERBOSE_SUPER;

tools/testing/selftests/bpf/test_progs.c

1099

env_verbosity = env->verbosity;

tools/testing/selftests/bpf/test_progs.c

110

env.subtest_state->stdout_saved = NULL;

tools/testing/selftests/bpf/test_progs.c

111

stdout = env.test_state->stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1112

env->get_test_cnt = true;

tools/testing/selftests/bpf/test_progs.c

1115

env->list_test_names = true;

tools/testing/selftests/bpf/test_progs.c

1119

env->workers = atoi(arg);

tools/testing/selftests/bpf/test_progs.c

112

stderr = env.test_state->stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1120

if (!env->workers) {

tools/testing/selftests/bpf/test_progs.c

1125

env->workers = get_nprocs();

tools/testing/selftests/bpf/test_progs.c

1129

env->debug = true;

tools/testing/selftests/bpf/test_progs.c

1132

env->json = fopen(arg, "w");

tools/testing/selftests/bpf/test_progs.c

1133

if (env->json == NULL) {

tools/testing/selftests/bpf/test_progs.c

114

if (env.test_state->stdout_saved)

tools/testing/selftests/bpf/test_progs.c

1147

&env->tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

115

fclose(env.test_state->stdout_saved);

tools/testing/selftests/bpf/test_progs.c

1151

&env->tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

1156

env->secs_till_kill = atoi(arg);

tools/testing/selftests/bpf/test_progs.c

1157

if (env->secs_till_kill < 0) {

tools/testing/selftests/bpf/test_progs.c

116

env.test_state->stdout_saved = NULL;

tools/testing/selftests/bpf/test_progs.c

1161

if (env->secs_till_kill < env->secs_till_notify) {

tools/testing/selftests/bpf/test_progs.c

1162

env->secs_till_notify = 0;

tools/testing/selftests/bpf/test_progs.c

117

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

118

stderr = env.stderr_saved;

tools/testing/selftests/bpf/test_progs.c

1298

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1299

stderr = env.stderr_saved;

tools/testing/selftests/bpf/test_progs.c

1301

if (env.test) {

tools/testing/selftests/bpf/test_progs.c

1302

env.test_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

1303

dump_test_log(env.test, env.test_state, true, false, NULL);

tools/testing/selftests/bpf/test_progs.c

1318

if (env.worker_id != -1)

tools/testing/selftests/bpf/test_progs.c

1319

fprintf(stderr, "[%d]: ", env.worker_id);

tools/testing/selftests/bpf/test_progs.c

1350

for (i = 0; i < env.workers; i++)

tools/testing/selftests/bpf/test_progs.c

1351

if (env.worker_socks[i] > 0)

tools/testing/selftests/bpf/test_progs.c

1352

close(env.worker_socks[i]);

tools/testing/selftests/bpf/test_progs.c

1395

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1408

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1428

env.test = test;

tools/testing/selftests/bpf/test_progs.c

1429

env.test_state = state;

tools/testing/selftests/bpf/test_progs.c

1444

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

1451

if (verbose() && env.worker_id == -1)

tools/testing/selftests/bpf/test_progs.c

1584

env.worker_current_test[data->worker_id] = test_to_run;

tools/testing/selftests/bpf/test_progs.c

160

struct test_env env = {};

tools/testing/selftests/bpf/test_progs.c

1624

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1633

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1641

static void calculate_summary_and_print_errors(struct test_env *env)

tools/testing/selftests/bpf/test_progs.c

1663

if (env->json) {

tools/testing/selftests/bpf/test_progs.c

1664

w = jsonw_new(env->json);

tools/testing/selftests/bpf/test_progs.c

1666

fprintf(env->stderr_saved, "Failed to create new JSON stream.");

tools/testing/selftests/bpf/test_progs.c

1705

if (env->json)

tools/testing/selftests/bpf/test_progs.c

1706

fclose(env->json);

tools/testing/selftests/bpf/test_progs.c

1708

if (env->not_built_cnt)

tools/testing/selftests/bpf/test_progs.c

1710

succ_cnt, sub_succ_cnt, skip_cnt, env->not_built_cnt,

tools/testing/selftests/bpf/test_progs.c

1716

env->succ_cnt = succ_cnt;

tools/testing/selftests/bpf/test_progs.c

1717

env->sub_succ_cnt = sub_succ_cnt;

tools/testing/selftests/bpf/test_progs.c

1718

env->fail_cnt = fail_cnt;

tools/testing/selftests/bpf/test_progs.c

1719

env->skip_cnt = skip_cnt;

tools/testing/selftests/bpf/test_progs.c

1734

dispatcher_threads = calloc(sizeof(pthread_t), env.workers);

tools/testing/selftests/bpf/test_progs.c

1735

data = calloc(sizeof(struct dispatch_data), env.workers);

tools/testing/selftests/bpf/test_progs.c

1737

env.worker_current_test = calloc(sizeof(int), env.workers);

tools/testing/selftests/bpf/test_progs.c

1738

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1742

data[i].sock_fd = env.worker_socks[i];

tools/testing/selftests/bpf/test_progs.c

1751

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1758

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1760

i, env.worker_current_test[i] + 1);

tools/testing/selftests/bpf/test_progs.c

1770

free(env.worker_current_test);

tools/testing/selftests/bpf/test_progs.c

1793

env.not_built_cnt++;

tools/testing/selftests/bpf/test_progs.c

1802

calculate_summary_and_print_errors(&env);

tools/testing/selftests/bpf/test_progs.c

1805

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1808

pid = waitpid(env.worker_pids[i], &wstatus, 0);

tools/testing/selftests/bpf/test_progs.c

1809

if (pid != env.worker_pids[i])

tools/testing/selftests/bpf/test_progs.c

1913

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1915

env.worker_id);

tools/testing/selftests/bpf/test_progs.c

1923

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1925

env.worker_id,

tools/testing/selftests/bpf/test_progs.c

1963

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1965

env.worker_id,

tools/testing/selftests/bpf/test_progs.c

1971

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1972

fprintf(stderr, "[%d]: unknown message.\n", env.worker_id);

tools/testing/selftests/bpf/test_progs.c

200

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

202

env.test->test_name, env.subtest_state->name);

tools/testing/selftests/bpf/test_progs.c

2021

env.stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

2022

env.stderr_saved = stderr;

tools/testing/selftests/bpf/test_progs.c

2024

env.secs_till_notify = 10;

tools/testing/selftests/bpf/test_progs.c

2025

env.secs_till_kill = 120;

tools/testing/selftests/bpf/test_progs.c

2026

err = argp_parse(&argp, argc, argv, 0, NULL, &env);

tools/testing/selftests/bpf/test_progs.c

2048

env.jit_enabled = is_jit_enabled();

tools/testing/selftests/bpf/test_progs.c

2049

env.nr_cpus = libbpf_num_possible_cpus();

tools/testing/selftests/bpf/test_progs.c

205

env.test->test_name);

tools/testing/selftests/bpf/test_progs.c

2050

if (env.nr_cpus < 0) {

tools/testing/selftests/bpf/test_progs.c

2052

env.nr_cpus);

tools/testing/selftests/bpf/test_progs.c

2056

env.has_testmod = true;

tools/testing/selftests/bpf/test_progs.c

2057

if (!env.list_test_names) {

tools/testing/selftests/bpf/test_progs.c

2062

fprintf(env.stderr_saved, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");

tools/testing/selftests/bpf/test_progs.c

2063

env.has_testmod = false;

tools/testing/selftests/bpf/test_progs.c

207

switch (env.watchdog_state) {

tools/testing/selftests/bpf/test_progs.c

2072

test->selected = should_run(&env.test_selector,

tools/testing/selftests/bpf/test_progs.c

2087

test->should_tmon = should_tmon(&env.tmon_selector, test->test_name);

tools/testing/selftests/bpf/test_progs.c

209

fprintf(env.stderr_saved, "WATCHDOG: test case %s executes for %d seconds...\n",

tools/testing/selftests/bpf/test_progs.c

2091

if (env.get_test_cnt || env.list_test_names)

tools/testing/selftests/bpf/test_progs.c

2092

env.workers = 0;

tools/testing/selftests/bpf/test_progs.c

2095

env.worker_id = -1; /* main process */

tools/testing/selftests/bpf/test_progs.c

2096

if (env.workers) {

tools/testing/selftests/bpf/test_progs.c

2097

env.worker_pids = calloc(sizeof(pid_t), env.workers);

tools/testing/selftests/bpf/test_progs.c

2098

env.worker_socks = calloc(sizeof(int), env.workers);

tools/testing/selftests/bpf/test_progs.c

2099

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

210

test_name, env.secs_till_notify);

tools/testing/selftests/bpf/test_progs.c

2100

fprintf(stdout, "Launching %d workers.\n", env.workers);

tools/testing/selftests/bpf/test_progs.c

2101

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

211

timeout.it_value.tv_sec = env.secs_till_kill - env.secs_till_notify;

tools/testing/selftests/bpf/test_progs.c

2115

env.worker_pids[i] = pid;

tools/testing/selftests/bpf/test_progs.c

2116

env.worker_socks[i] = sv[0];

tools/testing/selftests/bpf/test_progs.c

2119

env.worker_id = i;

tools/testing/selftests/bpf/test_progs.c

212

env.watchdog_state = WD_KILL;

tools/testing/selftests/bpf/test_progs.c

2124

if (env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

213

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

2141

!env.get_test_cnt && !env.list_test_names) {

tools/testing/selftests/bpf/test_progs.c

2144

env.not_built_cnt++;

tools/testing/selftests/bpf/test_progs.c

215

fprintf(env.stderr_saved, "Failed to arm watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

2150

if (env.get_test_cnt) {

tools/testing/selftests/bpf/test_progs.c

2151

env.succ_cnt++;

tools/testing/selftests/bpf/test_progs.c

2155

if (env.list_test_names) {

tools/testing/selftests/bpf/test_progs.c

2156

fprintf(env.stdout_saved, "%s\n", test->test_name);

tools/testing/selftests/bpf/test_progs.c

2157

env.succ_cnt++;

tools/testing/selftests/bpf/test_progs.c

2164

if (env.get_test_cnt) {

tools/testing/selftests/bpf/test_progs.c

2165

printf("%d\n", env.succ_cnt);

tools/testing/selftests/bpf/test_progs.c

2169

if (env.list_test_names)

tools/testing/selftests/bpf/test_progs.c

2172

calculate_summary_and_print_errors(&env);

tools/testing/selftests/bpf/test_progs.c

2174

close(env.saved_netns_fd);

tools/testing/selftests/bpf/test_progs.c

2176

if (!env.list_test_names && env.has_testmod)

tools/testing/selftests/bpf/test_progs.c

2179

free_test_selector(&env.test_selector);

tools/testing/selftests/bpf/test_progs.c

218

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

2180

free_test_selector(&env.subtest_selector);

tools/testing/selftests/bpf/test_progs.c

2181

free_test_selector(&env.tmon_selector);

tools/testing/selftests/bpf/test_progs.c

2184

if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0)

tools/testing/selftests/bpf/test_progs.c

2187

return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;

tools/testing/selftests/bpf/test_progs.c

220

test_name, env.secs_till_kill);

tools/testing/selftests/bpf/test_progs.c

221

pthread_kill(env.main_thread, SIGSEGV);

tools/testing/selftests/bpf/test_progs.c

231

if (env.secs_till_kill == 0)

tools/testing/selftests/bpf/test_progs.c

233

if (env.secs_till_notify > 0) {

tools/testing/selftests/bpf/test_progs.c

234

env.watchdog_state = WD_NOTIFY;

tools/testing/selftests/bpf/test_progs.c

235

timeout.it_value.tv_sec = env.secs_till_notify;

tools/testing/selftests/bpf/test_progs.c

237

env.watchdog_state = WD_KILL;

tools/testing/selftests/bpf/test_progs.c

238

timeout.it_value.tv_sec = env.secs_till_kill;

tools/testing/selftests/bpf/test_progs.c

240

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

242

fprintf(env.stderr_saved, "Failed to start watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

250

env.watchdog_state = WD_NOTIFY;

tools/testing/selftests/bpf/test_progs.c

251

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

253

fprintf(env.stderr_saved, "Failed to stop watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

264

env.main_thread = pthread_self();

tools/testing/selftests/bpf/test_progs.c

265

err = timer_create(CLOCK_MONOTONIC, &watchdog_sev, &env.watchdog);

tools/testing/selftests/bpf/test_progs.c

374

fprintf(env.stdout_saved, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name);

tools/testing/selftests/bpf/test_progs.c

376

fprintf(env.stdout_saved, "FAIL");

tools/testing/selftests/bpf/test_progs.c

378

fprintf(env.stdout_saved, "SKIP (not built)");

tools/testing/selftests/bpf/test_progs.c

380

fprintf(env.stdout_saved, "OK");

tools/testing/selftests/bpf/test_progs.c

382

fprintf(env.stdout_saved, "SKIP");

tools/testing/selftests/bpf/test_progs.c

384

fprintf(env.stdout_saved, "OK (SKIP: %d/%d)", skipped_cnt, subtests_cnt);

tools/testing/selftests/bpf/test_progs.c

386

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

392

fprintf(env.stdout_saved, "%s", log_buf);

tools/testing/selftests/bpf/test_progs.c

394

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

405

fprintf(env.stdout_saved, "#%-*s %s/%s",

tools/testing/selftests/bpf/test_progs.c

410

fprintf(env.stdout_saved, ":%s", result);

tools/testing/selftests/bpf/test_progs.c

412

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

443

if (env.worker_id != -1)

tools/testing/selftests/bpf/test_progs.c

48

return env.verbosity > VERBOSE_NONE;

tools/testing/selftests/bpf/test_progs.c

511

for (i = 0; i < env.nr_cpus; i++)

tools/testing/selftests/bpf/test_progs.c

528

env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY);

tools/testing/selftests/bpf/test_progs.c

529

if (env.saved_netns_fd == -1) {

tools/testing/selftests/bpf/test_progs.c

537

if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {

tools/testing/selftests/bpf/test_progs.c

54

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

545

struct prog_test_def *test = env.test;

tools/testing/selftests/bpf/test_progs.c

546

struct test_state *test_state = env.test_state;

tools/testing/selftests/bpf/test_progs.c

547

struct subtest_state *subtest_state = env.subtest_state;

tools/testing/selftests/bpf/test_progs.c

558

if (verbose() && !env.workers)

tools/testing/selftests/bpf/test_progs.c

566

env.subtest_state = NULL;

tools/testing/selftests/bpf/test_progs.c

571

struct prog_test_def *test = env.test;

tools/testing/selftests/bpf/test_progs.c

572

struct test_state *state = env.test_state;

tools/testing/selftests/bpf/test_progs.c

577

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

594

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

604

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

610

if (!should_run_subtest(&env.test_selector,

tools/testing/selftests/bpf/test_progs.c

611

&env.subtest_selector,

tools/testing/selftests/bpf/test_progs.c

620

subtest_state->should_tmon = match_subtest_desc(&env.tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

624

env.subtest_state = subtest_state;

tools/testing/selftests/bpf/test_progs.c

638

env.test_state->force_log = true;

tools/testing/selftests/bpf/test_progs.c

64

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

643

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

644

env.subtest_state->skipped = true;

tools/testing/selftests/bpf/test_progs.c

646

env.test_state->skip_cnt++;

tools/testing/selftests/bpf/test_progs.c

651

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

652

env.subtest_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

654

env.test_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

661

if (!env.test->need_cgroup_cleanup) {

tools/testing/selftests/bpf/test_progs.c

665

env.test->test_num, env.test->test_name);

tools/testing/selftests/bpf/test_progs.c

669

env.test->need_cgroup_cleanup = true;

tools/testing/selftests/bpf/test_progs.c

676

env.test->test_num, env.test->test_name, path, errno);

tools/testing/selftests/bpf/test_progs.c

683

env.test->test_num, env.test->test_name, path, errno);

tools/testing/selftests/bpf/test_progs.c

69

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

70

env.subtest_state->stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

72

env.test_state->stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

808

if (env.test->should_tmon ||

tools/testing/selftests/bpf/test_progs.c

809

(env.subtest_state && env.subtest_state->should_tmon)) {

tools/testing/selftests/bpf/test_progs.c

81

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

810

test_name = env.test->test_name;

tools/testing/selftests/bpf/test_progs.c

811

subtest_name = env.subtest_state ? env.subtest_state->name : NULL;

tools/testing/selftests/bpf/test_progs.c

86

env.stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

87

env.stderr_saved = stderr;

tools/testing/selftests/bpf/test_progs.c

98

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

994

if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)

tools/testing/selftests/bpf/test_progs.h

182

extern struct test_env env;

tools/testing/selftests/bpf/test_sockmap.c

126

struct test_env env;

tools/testing/selftests/bpf/test_sockmap.c

152

env.subtest_num++;

tools/testing/selftests/bpf/test_sockmap.c

157

env.fail_cnt++;

tools/testing/selftests/bpf/test_sockmap.c

162

env.succ_cnt++;

tools/testing/selftests/bpf/test_sockmap.c

179

env.type = o->map;

tools/testing/selftests/bpf/test_sockmap.c

180

env.subtest = t->title;

tools/testing/selftests/bpf/test_sockmap.c

181

env.prepend = o->prepend;

tools/testing/selftests/bpf/test_sockmap.c

182

env.test_num++;

tools/testing/selftests/bpf/test_sockmap.c

183

env.subtest_num = 0;

tools/testing/selftests/bpf/test_sockmap.c

184

env.fail_last = env.fail_cnt;

tools/testing/selftests/bpf/test_sockmap.c

191

int error = env.fail_cnt - env.fail_last;

tools/testing/selftests/bpf/test_sockmap.c

192

int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);

tools/testing/selftests/bpf/test_sockmap.c

198

env.test_num, env.subtest_num,

tools/testing/selftests/bpf/test_sockmap.c

200

env.prepend ? : "",

tools/testing/selftests/bpf/test_sockmap.c

201

env.subtest, error ? "FAIL" : "OK");

tools/testing/selftests/bpf/test_sockmap.c

207

env.succ_cnt, env.fail_cnt);

tools/testing/selftests/bpf/veristat.c

1122

for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {

tools/testing/selftests/bpf/veristat.c

1315

if (!env.quiet) {

tools/testing/selftests/bpf/veristat.c

1321

if (!env.quiet) {

tools/testing/selftests/bpf/veristat.c

1364

for (i = 0; i < env.output_spec.spec_cnt; i++)

tools/testing/selftests/bpf/veristat.c

1365

if (env.output_spec.ids[i] == id)

tools/testing/selftests/bpf/veristat.c

1428

close(env.memory_peak_fd);

tools/testing/selftests/bpf/veristat.c

1430

if (env.orig_cgroup[0]) {

tools/testing/selftests/bpf/veristat.c

1431

snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.orig_cgroup);

tools/testing/selftests/bpf/veristat.c

1434

log_errno("moving self to original cgroup %s\n", env.orig_cgroup);

tools/testing/selftests/bpf/veristat.c

1437

if (env.stat_cgroup[0]) {

tools/testing/selftests/bpf/veristat.c

1438

err = rmdir(env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1440

log_errno("deletion of cgroup %s", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1443

env.memory_peak_fd = -1;

tools/testing/selftests/bpf/veristat.c

1444

env.orig_cgroup[0] = 0;

tools/testing/selftests/bpf/veristat.c

1445

env.stat_cgroup[0] = 0;

tools/testing/selftests/bpf/veristat.c

1458

env.memory_peak_fd = -1;

tools/testing/selftests/bpf/veristat.c

1468

else if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1478

else if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1483

snprintf(env.orig_cgroup, sizeof(env.orig_cgroup), "%s/%s", cgroup_fs_mount, buf);

tools/testing/selftests/bpf/veristat.c

1491

strcpy(env.stat_cgroup, buf);

tools/testing/selftests/bpf/veristat.c

1493

snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1500

snprintf(buf, sizeof(buf), "%s/memory.peak", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1501

env.memory_peak_fd = open(buf, O_RDWR | O_APPEND);

tools/testing/selftests/bpf/veristat.c

1502

if (env.memory_peak_fd < 0) {

tools/testing/selftests/bpf/veristat.c

1510

if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1521

if (env.memory_peak_fd < 0)

tools/testing/selftests/bpf/veristat.c

1524

err = pread(env.memory_peak_fd, buf, sizeof(buf) - 1, 0);

tools/testing/selftests/bpf/veristat.c

1526

log_errno("pread(%s/memory.peak)", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1534

log_errno("%s/memory.peak:strtoll(%s)", env.stat_cgroup, buf);

tools/testing/selftests/bpf/veristat.c

1546

if (env.memory_peak_fd < 0)

tools/testing/selftests/bpf/veristat.c

1549

err = pwrite(env.memory_peak_fd, buf, sizeof(buf), 0);

tools/testing/selftests/bpf/veristat.c

1551

log_errno("pwrite(%s/memory.peak)", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1632

env.progs_skipped++;

tools/testing/selftests/bpf/veristat.c

1636

tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));

tools/testing/selftests/bpf/veristat.c

1639

env.prog_stats = tmp;

tools/testing/selftests/bpf/veristat.c

1640

stats = &env.prog_stats[env.prog_stat_cnt++];

tools/testing/selftests/bpf/veristat.c

1643

if (env.verbose || env.top_src_lines > 0) {

tools/testing/selftests/bpf/veristat.c

1644

buf_sz = env.log_size ? env.log_size : max_verifier_log_size();

tools/testing/selftests/bpf/veristat.c

1649

log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);

tools/testing/selftests/bpf/veristat.c

1651

if (env.top_src_lines > 0 && env.log_level == 0)

tools/testing/selftests/bpf/veristat.c

1657

log_level = 4 | (env.log_fixed ? 8 : 0);

tools/testing/selftests/bpf/veristat.c

1664

if (env.force_checkpoints)

tools/testing/selftests/bpf/veristat.c

1666

if (env.force_reg_invariants)

tools/testing/selftests/bpf/veristat.c

1678

if (env.verbose)

tools/testing/selftests/bpf/veristat.c

1685

env.progs_processed++;

tools/testing/selftests/bpf/veristat.c

1698

if (env.dump_mode & DUMP_JITED)

tools/testing/selftests/bpf/veristat.c

1700

if (env.dump_mode & DUMP_XLATED)

tools/testing/selftests/bpf/veristat.c

1706

if (env.verbose) {

tools/testing/selftests/bpf/veristat.c

1711

if (env.top_src_lines > 0)

tools/testing/selftests/bpf/veristat.c

1835

err = append_var_preset(&env.presets, &env.npresets, buf);

tools/testing/selftests/bpf/veristat.c

2207

if (env.verbose)

tools/testing/selftests/bpf/veristat.c

2209

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2213

if (env.verbose)

tools/testing/selftests/bpf/veristat.c

2215

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2219

if (!env.quiet && env.out_fmt == RESFMT_TABLE)

tools/testing/selftests/bpf/veristat.c

2232

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2237

env.files_processed++;

tools/testing/selftests/bpf/veristat.c

2246

err = set_global_vars(obj, env.presets, env.npresets);

tools/testing/selftests/bpf/veristat.c

2253

if (err && env.verbose) /* run process_prog() anyway to output per program failures */

tools/testing/selftests/bpf/veristat.c

2317

for (i = 0; i < env.sort_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2318

cmp = cmp_stat(s1, s2, env.sort_spec.ids[i],

tools/testing/selftests/bpf/veristat.c

2319

env.sort_spec.asc[i], env.sort_spec.abs[i]);

tools/testing/selftests/bpf/veristat.c

238

} env;

tools/testing/selftests/bpf/veristat.c

2416

for (i = 0; i < env.sort_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2418

env.sort_spec.ids[i],

tools/testing/selftests/bpf/veristat.c

2419

env.sort_spec.variants[i],

tools/testing/selftests/bpf/veristat.c

242

if (!env.verbose)

tools/testing/selftests/bpf/veristat.c

2420

env.sort_spec.asc[i],

tools/testing/selftests/bpf/veristat.c

2421

env.sort_spec.abs[i]);

tools/testing/selftests/bpf/veristat.c

244

if (level == LIBBPF_DEBUG && !env.debug)

tools/testing/selftests/bpf/veristat.c

2440

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2441

len = env.output_spec.lens[i];

tools/testing/selftests/bpf/veristat.c

2455

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2456

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2457

int *max_len = &env.output_spec.lens[i];

tools/testing/selftests/bpf/veristat.c

2468

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2473

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2534

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2535

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2536

int *max_len = &env.output_spec.lens[i], len;

tools/testing/selftests/bpf/veristat.c

2556

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2564

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2573

env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped);

tools/testing/selftests/bpf/veristat.c

2778

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2779

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2783

int len = env.output_spec.lens[3 * i + j];

tools/testing/selftests/bpf/veristat.c

2800

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2801

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2806

int *max_len = &env.output_spec.lens[3 * i + j];

tools/testing/selftests/bpf/veristat.c

2807

bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);

tools/testing/selftests/bpf/veristat.c

2846

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2847

int id = env.output_spec.ids[i], len;

tools/testing/selftests/bpf/veristat.c

2848

int *max_len_base = &env.output_spec.lens[3 * i + 0];

tools/testing/selftests/bpf/veristat.c

2849

int *max_len_comp = &env.output_spec.lens[3 * i + 1];

tools/testing/selftests/bpf/veristat.c

2850

int *max_len_diff = &env.output_spec.lens[3 * i + 2];

tools/testing/selftests/bpf/veristat.c

2924

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2934

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2983

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2984

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

2992

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2993

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

3013

if (env.filename_cnt != 2) {

tools/testing/selftests/bpf/veristat.c

3019

err = parse_stats_csv(env.filenames[0], &base_specs,

tools/testing/selftests/bpf/veristat.c

3020

&env.baseline_stats, &env.baseline_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3022

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);

tools/testing/selftests/bpf/veristat.c

3025

err = parse_stats_csv(env.filenames[1], &comp_specs,

tools/testing/selftests/bpf/veristat.c

3026

&env.prog_stats, &env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3028

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);

tools/testing/selftests/bpf/veristat.c

3038

env.filenames[0], env.filenames[1],

tools/testing/selftests/bpf/veristat.c

3045

env.filenames[0], env.filenames[1],

tools/testing/selftests/bpf/veristat.c

3056

tmp_sort_spec = env.sort_spec;

tools/testing/selftests/bpf/veristat.c

3057

env.sort_spec = join_sort_spec;

tools/testing/selftests/bpf/veristat.c

3058

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3059

qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3060

env.sort_spec = tmp_sort_spec;

tools/testing/selftests/bpf/veristat.c

3068

while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {

tools/testing/selftests/bpf/veristat.c

3074

base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;

tools/testing/selftests/bpf/veristat.c

3075

comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;

tools/testing/selftests/bpf/veristat.c

3079

i, env.filenames[0]);

tools/testing/selftests/bpf/veristat.c

3084

j, env.filenames[1]);

tools/testing/selftests/bpf/veristat.c

3088

tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats));

tools/testing/selftests/bpf/veristat.c

3091

env.join_stats = tmp;

tools/testing/selftests/bpf/veristat.c

3093

join = &env.join_stats[env.join_stat_cnt];

tools/testing/selftests/bpf/veristat.c

3121

env.join_stat_cnt += 1;

tools/testing/selftests/bpf/veristat.c

3125

qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats);

tools/testing/selftests/bpf/veristat.c

3132

if (env.out_fmt == RESFMT_TABLE)

tools/testing/selftests/bpf/veristat.c

3135

cur_fmt = env.out_fmt;

tools/testing/selftests/bpf/veristat.c

3142

for (i = 0; i < env.join_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3143

const struct verif_stats_join *join = &env.join_stats[i];

tools/testing/selftests/bpf/veristat.c

3148

if (env.top_n && cnt >= env.top_n)

tools/testing/selftests/bpf/veristat.c

3192

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3193

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

3201

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3202

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

3220

if (env.out_fmt == RESFMT_TABLE) {

tools/testing/selftests/bpf/veristat.c

3223

for (i = 0; i < env.prog_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3224

stats = &env.prog_stats[i];

tools/testing/selftests/bpf/veristat.c

3233

output_headers(env.out_fmt);

tools/testing/selftests/bpf/veristat.c

3234

for (i = 0; i < env.prog_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3235

stats = &env.prog_stats[i];

tools/testing/selftests/bpf/veristat.c

3238

if (env.top_n && cnt >= env.top_n)

tools/testing/selftests/bpf/veristat.c

3240

output_stats(stats, env.out_fmt, i == last_stat_idx);

tools/testing/selftests/bpf/veristat.c

3249

if (env.filename_cnt == 0) {

tools/testing/selftests/bpf/veristat.c

3256

for (i = 0; i < env.filename_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3257

err = process_obj(env.filenames[i]);

tools/testing/selftests/bpf/veristat.c

3259

fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);

tools/testing/selftests/bpf/veristat.c

3262

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3275

if (env.filename_cnt != 1) {

tools/testing/selftests/bpf/veristat.c

328

env.show_version = true;

tools/testing/selftests/bpf/veristat.c

3281

err = parse_stats_csv(env.filenames[0], &specs,

tools/testing/selftests/bpf/veristat.c

3282

&env.prog_stats, &env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3284

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);

tools/testing/selftests/bpf/veristat.c

3288

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3302

if (env.show_version) {

tools/testing/selftests/bpf/veristat.c

3307

if (env.verbose && env.quiet) {

tools/testing/selftests/bpf/veristat.c

331

env.verbose = true;

tools/testing/selftests/bpf/veristat.c

3312

if (env.verbose && env.log_level == 0)

tools/testing/selftests/bpf/veristat.c

3313

env.log_level = 1;

tools/testing/selftests/bpf/veristat.c

3315

if (env.output_spec.spec_cnt == 0) {

tools/testing/selftests/bpf/veristat.c

3316

if (env.out_fmt == RESFMT_CSV)

tools/testing/selftests/bpf/veristat.c

3317

env.output_spec = default_csv_output_spec;

tools/testing/selftests/bpf/veristat.c

3319

env.output_spec = default_output_spec;

tools/testing/selftests/bpf/veristat.c

3321

if (env.sort_spec.spec_cnt == 0)

tools/testing/selftests/bpf/veristat.c

3322

env.sort_spec = default_sort_spec;

tools/testing/selftests/bpf/veristat.c

3324

if (env.comparison_mode && env.replay_mode) {

tools/testing/selftests/bpf/veristat.c

3330

if (env.comparison_mode)

tools/testing/selftests/bpf/veristat.c

3332

else if (env.replay_mode)

tools/testing/selftests/bpf/veristat.c

3337

free_verif_stats(env.prog_stats, env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3338

free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3339

free(env.join_stats);

tools/testing/selftests/bpf/veristat.c

334

env.debug = true;

tools/testing/selftests/bpf/veristat.c

3340

for (i = 0; i < env.filename_cnt; i++)

tools/testing/selftests/bpf/veristat.c

3341

free(env.filenames[i]);

tools/testing/selftests/bpf/veristat.c

3342

free(env.filenames);

tools/testing/selftests/bpf/veristat.c

3343

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3344

free(env.allow_filters[i].any_glob);

tools/testing/selftests/bpf/veristat.c

3345

free(env.allow_filters[i].file_glob);

tools/testing/selftests/bpf/veristat.c

3346

free(env.allow_filters[i].prog_glob);

tools/testing/selftests/bpf/veristat.c

3348

free(env.allow_filters);

tools/testing/selftests/bpf/veristat.c

3349

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

335

env.verbose = true;

tools/testing/selftests/bpf/veristat.c

3350

free(env.deny_filters[i].any_glob);

tools/testing/selftests/bpf/veristat.c

3351

free(env.deny_filters[i].file_glob);

tools/testing/selftests/bpf/veristat.c

3352

free(env.deny_filters[i].prog_glob);

tools/testing/selftests/bpf/veristat.c

3354

free(env.deny_filters);

tools/testing/selftests/bpf/veristat.c

3355

for (i = 0; i < env.npresets; ++i) {

tools/testing/selftests/bpf/veristat.c

3356

free(env.presets[i].full_name);

tools/testing/selftests/bpf/veristat.c

3357

for (j = 0; j < env.presets[i].atom_count; ++j) {

tools/testing/selftests/bpf/veristat.c

3358

switch (env.presets[i].atoms[j].type) {

tools/testing/selftests/bpf/veristat.c

3360

free(env.presets[i].atoms[j].name);

tools/testing/selftests/bpf/veristat.c

3363

if (env.presets[i].atoms[j].index.type == ENUMERATOR)

tools/testing/selftests/bpf/veristat.c

3364

free(env.presets[i].atoms[j].index.svalue);

tools/testing/selftests/bpf/veristat.c

3368

free(env.presets[i].atoms);

tools/testing/selftests/bpf/veristat.c

3369

if (env.presets[i].value.type == ENUMERATOR)

tools/testing/selftests/bpf/veristat.c

3370

free(env.presets[i].value.svalue);

tools/testing/selftests/bpf/veristat.c

3372

free(env.presets);

tools/testing/selftests/bpf/veristat.c

338

env.quiet = true;

tools/testing/selftests/bpf/veristat.c

341

err = parse_stats(arg, &env.output_spec);

tools/testing/selftests/bpf/veristat.c

346

err = parse_stats(arg, &env.sort_spec);

tools/testing/selftests/bpf/veristat.c

352

env.out_fmt = RESFMT_TABLE;

tools/testing/selftests/bpf/veristat.c

354

env.out_fmt = RESFMT_CSV;

tools/testing/selftests/bpf/veristat.c

362

env.log_level = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

369

env.log_fixed = true;

tools/testing/selftests/bpf/veristat.c

373

env.log_size = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

380

env.force_checkpoints = true;

tools/testing/selftests/bpf/veristat.c

383

env.force_reg_invariants = true;

tools/testing/selftests/bpf/veristat.c

387

env.top_n = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

394

env.comparison_mode = true;

tools/testing/selftests/bpf/veristat.c

397

env.replay_mode = true;

tools/testing/selftests/bpf/veristat.c

403

err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);

tools/testing/selftests/bpf/veristat.c

405

err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);

tools/testing/selftests/bpf/veristat.c

413

env.top_src_lines = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

423

err = append_var_preset(&env.presets, &env.npresets, arg);

tools/testing/selftests/bpf/veristat.c

442

env.dump_mode |= DUMP_XLATED;

tools/testing/selftests/bpf/veristat.c

444

env.dump_mode |= DUMP_JITED;

tools/testing/selftests/bpf/veristat.c

522

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

523

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

537

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

538

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

729

err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);

tools/testing/selftests/bpf/veristat.c

731

err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);

tools/testing/selftests/bpf/veristat.c

753

tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));

tools/testing/selftests/bpf/veristat.c

756

env.filenames = tmp;

tools/testing/selftests/bpf/veristat.c

757

env.filenames[env.filename_cnt] = strdup(path);

tools/testing/selftests/bpf/veristat.c

758

if (!env.filenames[env.filename_cnt])

tools/testing/selftests/bpf/veristat.c

760

env.filename_cnt++;

tools/testing/selftests/bpf/xdp_features.c

100

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

101

env.feature.action = XDP_TX;

tools/testing/selftests/bpf/xdp_features.c

103

env.feature.drv_feature = NETDEV_XDP_ACT_REDIRECT;

tools/testing/selftests/bpf/xdp_features.c

104

env.feature.action = XDP_REDIRECT;

tools/testing/selftests/bpf/xdp_features.c

106

env.feature.drv_feature = NETDEV_XDP_ACT_NDO_XMIT;

tools/testing/selftests/bpf/xdp_features.c

116

switch (env.feature.action) {

tools/testing/selftests/bpf/xdp_features.c

131

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT)

tools/testing/selftests/bpf/xdp_features.c

141

env.verbosity = true;

tools/testing/selftests/bpf/xdp_features.c

144

env.is_tester = true;

tools/testing/selftests/bpf/xdp_features.c

155

&env.dut_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

164

&env.dut_ctrl_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

172

if (make_sockaddr(AF_INET6, arg, 0, &env.tester_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

187

env.ifindex = if_nametoindex(arg);

tools/testing/selftests/bpf/xdp_features.c

188

if (!env.ifindex)

tools/testing/selftests/bpf/xdp_features.c

189

env.ifindex = strtoul(arg, NULL, 0);

tools/testing/selftests/bpf/xdp_features.c

190

if (!env.ifindex || !if_indextoname(env.ifindex, env.ifname)) {

tools/testing/selftests/bpf/xdp_features.c

213

env.feature.drv_feature = NETDEV_XDP_ACT_NDO_XMIT;

tools/testing/selftests/bpf/xdp_features.c

214

env.feature.action = -EINVAL;

tools/testing/selftests/bpf/xdp_features.c

215

env.ifindex = -ENODEV;

tools/testing/selftests/bpf/xdp_features.c

216

strscpy(env.ifname, "unknown");

tools/testing/selftests/bpf/xdp_features.c

218

&env.dut_ctrl_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

220

&env.dut_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

221

make_sockaddr(AF_INET6, "::ffff:127.0.0.1", 0, &env.tester_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

262

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

271

env.ifname, strerror(-err));

tools/testing/selftests/bpf/xdp_features.c

281

enum xdp_action action = env.feature.action;

tools/testing/selftests/bpf/xdp_features.c

286

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT) {

tools/testing/selftests/bpf/xdp_features.c

288

.ifindex = env.ifindex,

tools/testing/selftests/bpf/xdp_features.c

333

err = bpf_xdp_attach(env.ifindex, bpf_program__fd(prog), flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

336

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

373

"Failed creating control socket on device %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

381

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

39

} env;

tools/testing/selftests/bpf/xdp_features.c

424

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

435

err = bpf_xdp_query(env.ifindex, XDP_FLAGS_DRV_MODE,

tools/testing/selftests/bpf/xdp_features.c

440

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

48

if (level == LIBBPF_DEBUG && !env.verbosity)

tools/testing/selftests/bpf/xdp_features.c

489

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

511

switch (env.feature.action) {

tools/testing/selftests/bpf/xdp_features.c

523

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT)

tools/testing/selftests/bpf/xdp_features.c

560

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

568

(struct sockaddr *)&env.dut_addr, sizeof(env.dut_addr));

tools/testing/selftests/bpf/xdp_features.c

593

err = connect(sockfd, (struct sockaddr *)&env.dut_ctrl_addr,

tools/testing/selftests/bpf/xdp_features.c

594

sizeof(env.dut_ctrl_addr));

tools/testing/selftests/bpf/xdp_features.c

610

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT ||

tools/testing/selftests/bpf/xdp_features.c

611

env.feature.action == XDP_TX)

tools/testing/selftests/bpf/xdp_features.c

616

err = bpf_xdp_attach(env.ifindex, bpf_program__fd(prog), flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

619

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

648

env.feature.drv_feature & advertised_feature ? GREEN("ADVERTISED")

tools/testing/selftests/bpf/xdp_features.c

651

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

674

if (env.ifindex < 0) {

tools/testing/selftests/bpf/xdp_features.c

675

fprintf(stderr, "Invalid device name %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

687

((struct sockaddr_in6 *)&env.tester_addr)->sin6_addr;

tools/testing/selftests/bpf/xdp_features.c

689

((struct sockaddr_in6 *)&env.dut_addr)->sin6_addr;

tools/testing/selftests/bpf/xdp_features.c

704

if (env.is_tester) {

tools/testing/selftests/bpf/xdp_features.c

707

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

711

fprintf(stdout, "Starting test on device %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

91

env.feature.action = XDP_PASS;

tools/testing/selftests/bpf/xdp_features.c

92

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

94

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

95

env.feature.action = XDP_DROP;

tools/testing/selftests/bpf/xdp_features.c

97

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

98

env.feature.action = XDP_ABORTED;

tools/testing/selftests/mm/mkdirty.c

32

static sigjmp_buf env;

tools/testing/selftests/mm/mkdirty.c

37

siglongjmp(env, 1);

tools/testing/selftests/mm/mkdirty.c

38

siglongjmp(env, 2);

tools/testing/selftests/mm/mkdirty.c

51

ret = sigsetjmp(env, 1);

tools/testing/selftests/powerpc/mm/wild_bctr.c

113

unsigned long env;

virt/kvm/kvm_main.c

6291

struct kobj_uevent_env *env;

virt/kvm/kvm_main.c

6308

env = kzalloc_obj(*env);

virt/kvm/kvm_main.c

6309

if (!env)

virt/kvm/kvm_main.c

6312

add_uevent_var(env, "CREATED=%llu", created);

virt/kvm/kvm_main.c

6313

add_uevent_var(env, "COUNT=%llu", active);

virt/kvm/kvm_main.c

6316

add_uevent_var(env, "EVENT=create");

virt/kvm/kvm_main.c

6319

add_uevent_var(env, "EVENT=destroy");

virt/kvm/kvm_main.c

6321

add_uevent_var(env, "PID=%d", kvm->userspace_pid);

virt/kvm/kvm_main.c

6329

add_uevent_var(env, "STATS_PATH=%s", tmp);

virt/kvm/kvm_main.c

6334

env->envp[env->envp_idx++] = NULL;

virt/kvm/kvm_main.c

6335

kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);

virt/kvm/kvm_main.c

6336

kfree(env);