arch/alpha/include/asm/err_ev7.h

182

struct ev7_pal_environmental_subpacket env; /* Type 10-16 */

arch/alpha/include/asm/err_ev7.h

196

struct ev7_pal_environmental_subpacket *env[7]; /* Type 10-16 */

arch/alpha/kernel/err_ev7.c

104

lf_subpackets->env[ev7_lf_env_index(subpacket->type)] =

arch/alpha/kernel/err_marvel.c

28

struct ev7_pal_environmental_subpacket *env;

arch/alpha/kernel/err_marvel.c

49

env = lf_subpackets->env[ev7_lf_env_index(ev_packets[i].type)];

arch/alpha/kernel/err_marvel.c

50

if (!env)

arch/alpha/kernel/err_marvel.c

56

env->cabinet,

arch/alpha/kernel/err_marvel.c

57

env->drawer);

arch/alpha/kernel/err_marvel.c

61

env->module_type,

arch/alpha/kernel/err_marvel.c

62

env->unit_id,

arch/alpha/kernel/err_marvel.c

63

env->condition);

arch/alpha/kernel/err_marvel.c

77

if (lf_subpackets->env[i])

arch/loongarch/kvm/main.c

294

unsigned long env, gcfg = 0;

arch/loongarch/kvm/main.c

296

env = read_csr_gcfg();

arch/loongarch/kvm/main.c

314

if (env & CSR_GCFG_GCIP_SECURE)

arch/loongarch/kvm/main.c

316

if (env & CSR_GCFG_MATP_ROOT)

arch/mips/alchemy/common/prom.c

67

char **env = prom_envp;

arch/mips/alchemy/common/prom.c

69

int yamon = (*env && strchr(*env, '=') == NULL);

arch/mips/alchemy/common/prom.c

71

while (*env) {

arch/mips/alchemy/common/prom.c

73

if (strcmp(envname, *env++) == 0)

arch/mips/alchemy/common/prom.c

74

return *env;

arch/mips/alchemy/common/prom.c

75

} else if (strncmp(envname, *env, i) == 0 && (*env)[i] == '=')

arch/mips/alchemy/common/prom.c

76

return *env + i + 1;

arch/mips/alchemy/common/prom.c

77

env++;

arch/mips/sgi-ip22/ip22-gio.c

201

static int gio_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

arch/mips/sgi-ip22/ip22-gio.c

205

add_uevent_var(env, "MODALIAS=gio:%x", gio_dev->id.id);

arch/parisc/kernel/drivers.c

566

static int parisc_uevent(const struct device *dev, struct kobj_uevent_env *env)

arch/parisc/kernel/drivers.c

578

if (add_uevent_var(env, "PARISC_NAME=%s", padev->name))

arch/parisc/kernel/drivers.c

582

if (add_uevent_var(env, "MODALIAS=%s", modalias))

arch/powerpc/include/asm/elf.h

192

unsigned long env;

arch/powerpc/include/asm/setjmp.h

12

extern int setjmp(jmp_buf env) __attribute__((returns_twice));

arch/powerpc/include/asm/setjmp.h

13

extern void longjmp(jmp_buf env, int val) __attribute__((noreturn));

arch/powerpc/platforms/ps3/system-bus.c

442

static int ps3_system_bus_uevent(const struct device *_dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/ps3/system-bus.c

446

if (add_uevent_var(env, "MODALIAS=ps3:%d:%d", dev->match_id,

arch/powerpc/platforms/pseries/ibmebus.c

431

static int ibmebus_bus_modalias(const struct device *dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/pseries/ibmebus.c

433

return of_device_uevent_modalias(dev, env);

arch/powerpc/platforms/pseries/vio.c

1590

static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env)

arch/powerpc/platforms/pseries/vio.c

1598

add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, cp);

arch/sparc/kernel/ldc.c

1612

p->env = (data_len |

arch/sparc/kernel/ldc.c

1622

p->env,

arch/sparc/kernel/ldc.c

1774

p->env,

arch/sparc/kernel/ldc.c

1816

pkt_len = p->env & LDC_LEN;

arch/sparc/kernel/ldc.c

1831

if ((first_frag == NULL && !(p->env & LDC_START)) ||

arch/sparc/kernel/ldc.c

1832

(first_frag != NULL && (p->env & LDC_START))) {

arch/sparc/kernel/ldc.c

1872

if (p->env & LDC_STOP)

arch/sparc/kernel/ldc.c

435

p->env = lp->cfg.mode;

arch/sparc/kernel/ldc.c

440

p->env, p->seqid);

arch/sparc/kernel/ldc.c

455

p->env = lp->cfg.mode;

arch/sparc/kernel/ldc.c

459

p->env, p->seqid);

arch/sparc/kernel/ldc.c

474

p->env = 0;

arch/sparc/kernel/ldc.c

479

p->env, p->seqid, p->u.r.ackid);

arch/sparc/kernel/ldc.c

61

u8 env;

arch/sparc/kernel/ldc.c

673

p->stype, p->seqid, p->env);

arch/sparc/kernel/ldc.c

677

p->env != lp->cfg.mode)

arch/sparc/kernel/ldc.c

693

p->stype, p->seqid, p->env);

arch/sparc/kernel/ldc.c

696

p->env != lp->cfg.mode)

arch/sparc/kernel/ldc.c

716

p->stype, p->seqid, p->env, p->u.r.ackid);

arch/sparc/kernel/vio.c

50

static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env)

arch/sparc/kernel/vio.c

54

add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, vio_dev->compat);

arch/um/drivers/port_user.c

171

char *env;

arch/um/drivers/port_user.c

176

if ((env = getenv("UML_PORT_HELPER")))

arch/um/drivers/port_user.c

177

argv[2] = env;

arch/um/drivers/port_user.c

187

if (env == NULL)

arch/x86/include/asm/fpu/signal.h

21

extern void convert_from_fxsr(struct user_i387_ia32_struct *env,

arch/x86/include/asm/fpu/signal.h

24

const struct user_i387_ia32_struct *env);

arch/x86/kernel/fpu/regset.c

331

static void __convert_from_fxsr(struct user_i387_ia32_struct *env,

arch/x86/kernel/fpu/regset.c

335

struct _fpreg *to = (struct _fpreg *) &env->st_space[0];

arch/x86/kernel/fpu/regset.c

339

env->cwd = fxsave->cwd | 0xffff0000u;

arch/x86/kernel/fpu/regset.c

340

env->swd = fxsave->swd | 0xffff0000u;

arch/x86/kernel/fpu/regset.c

341

env->twd = twd_fxsr_to_i387(fxsave);

arch/x86/kernel/fpu/regset.c

344

env->fip = fxsave->rip;

arch/x86/kernel/fpu/regset.c

345

env->foo = fxsave->rdp;

arch/x86/kernel/fpu/regset.c

350

env->fcs = task_pt_regs(tsk)->cs;

arch/x86/kernel/fpu/regset.c

352

savesegment(ds, env->fos);

arch/x86/kernel/fpu/regset.c

354

env->fos = tsk->thread.ds;

arch/x86/kernel/fpu/regset.c

356

env->fos |= 0xffff0000;

arch/x86/kernel/fpu/regset.c

358

env->fip = fxsave->fip;

arch/x86/kernel/fpu/regset.c

359

env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);

arch/x86/kernel/fpu/regset.c

360

env->foo = fxsave->foo;

arch/x86/kernel/fpu/regset.c

361

env->fos = fxsave->fos;

arch/x86/kernel/fpu/regset.c

369

convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)

arch/x86/kernel/fpu/regset.c

371

__convert_from_fxsr(env, tsk, &x86_task_fpu(tsk)->fpstate->regs.fxsave);

arch/x86/kernel/fpu/regset.c

375

const struct user_i387_ia32_struct *env)

arch/x86/kernel/fpu/regset.c

378

struct _fpreg *from = (struct _fpreg *) &env->st_space[0];

arch/x86/kernel/fpu/regset.c

382

fxsave->cwd = env->cwd;

arch/x86/kernel/fpu/regset.c

383

fxsave->swd = env->swd;

arch/x86/kernel/fpu/regset.c

384

fxsave->twd = twd_i387_to_fxsr(env->twd);

arch/x86/kernel/fpu/regset.c

385

fxsave->fop = (u16) ((u32) env->fcs >> 16);

arch/x86/kernel/fpu/regset.c

387

fxsave->rip = env->fip;

arch/x86/kernel/fpu/regset.c

388

fxsave->rdp = env->foo;

arch/x86/kernel/fpu/regset.c

391

fxsave->fip = env->fip;

arch/x86/kernel/fpu/regset.c

392

fxsave->fcs = (env->fcs & 0xffff);

arch/x86/kernel/fpu/regset.c

393

fxsave->foo = env->foo;

arch/x86/kernel/fpu/regset.c

394

fxsave->fos = env->fos;

arch/x86/kernel/fpu/regset.c

405

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/regset.c

428

__convert_from_fxsr(&env, target, fx);

arch/x86/kernel/fpu/regset.c

429

return membuf_write(&to, &env, sizeof(env));

arch/x86/kernel/fpu/regset.c

437

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/regset.c

447

ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);

arch/x86/kernel/fpu/regset.c

454

convert_to_fxsr(&fpu->fpstate->regs.fxsave, &env);

arch/x86/kernel/fpu/regset.c

456

memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env));

arch/x86/kernel/fpu/signal.c

327

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/signal.c

354

if (__copy_from_user(&env, buf, sizeof(env)))

arch/x86/kernel/fpu/signal.c

405

convert_to_fxsr(&fpregs->fxsave, &env);

arch/x86/kernel/fpu/signal.c

68

struct user_i387_ia32_struct env;

arch/x86/kernel/fpu/signal.c

76

convert_from_fxsr(&env, tsk);

arch/x86/kernel/fpu/signal.c

78

if (__copy_to_user(buf, &env, sizeof(env)) ||

block/genhd.c

1316

static int block_uevent(const struct device *dev, struct kobj_uevent_env *env)

block/genhd.c

1320

return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);

block/partitions/core.c

253

static int part_uevent(const struct device *dev, struct kobj_uevent_env *env)

block/partitions/core.c

257

add_uevent_var(env, "PARTN=%u", bdev_partno(part));

block/partitions/core.c

259

add_uevent_var(env, "PARTNAME=%s", part->bd_meta_info->volname);

block/partitions/core.c

261

add_uevent_var(env, "PARTUUID=%s", part->bd_meta_info->uuid);

drivers/acpi/bus.c

1110

static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/acpi/bus.c

1112

return __acpi_device_uevent_modalias(to_acpi_device(dev), env);

drivers/acpi/device_sysfs.c

241

struct kobj_uevent_env *env)

drivers/acpi/device_sysfs.c

251

if (add_uevent_var(env, "MODALIAS="))

drivers/acpi/device_sysfs.c

255

len = create_of_modalias(adev, &env->buf[env->buflen - 1],

drivers/acpi/device_sysfs.c

256

sizeof(env->buf) - env->buflen);

drivers/acpi/device_sysfs.c

258

len = create_pnp_modalias(adev, &env->buf[env->buflen - 1],

drivers/acpi/device_sysfs.c

259

sizeof(env->buf) - env->buflen);

drivers/acpi/device_sysfs.c

263

env->buflen += len;

drivers/acpi/device_sysfs.c

278

int acpi_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/acpi/device_sysfs.c

280

return __acpi_device_uevent_modalias(acpi_companion_match(dev), env);

drivers/acpi/internal.h

137

struct kobj_uevent_env *env);

drivers/amba/bus.c

239

static int amba_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/amba/bus.c

244

retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid);

drivers/amba/bus.c

248

retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid);

drivers/base/auxiliary.c

199

static int auxiliary_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/auxiliary.c

206

return add_uevent_var(env, "MODALIAS=%s%.*s", AUXILIARY_MODULE_PREFIX,

drivers/base/core.c

2639

static void dev_driver_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/core.c

2647

add_uevent_var(env, "DRIVER=%s", drv->name);

drivers/base/core.c

2654

static int dev_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

drivers/base/core.c

2667

add_uevent_var(env, "MAJOR=%u", MAJOR(dev->devt));

drivers/base/core.c

2668

add_uevent_var(env, "MINOR=%u", MINOR(dev->devt));

drivers/base/core.c

2671

add_uevent_var(env, "DEVNAME=%s", name);

drivers/base/core.c

2673

add_uevent_var(env, "DEVMODE=%#o", mode & 0777);

drivers/base/core.c

2675

add_uevent_var(env, "DEVUID=%u", from_kuid(&init_user_ns, uid));

drivers/base/core.c

2677

add_uevent_var(env, "DEVGID=%u", from_kgid(&init_user_ns, gid));

drivers/base/core.c

2683

add_uevent_var(env, "DEVTYPE=%s", dev->type->name);

drivers/base/core.c

2686

dev_driver_uevent(dev, env);

drivers/base/core.c

2689

of_device_uevent(dev, env);

drivers/base/core.c

2693

retval = dev->bus->uevent(dev, env);

drivers/base/core.c

2701

retval = dev->class->dev_uevent(dev, env);

drivers/base/core.c

2710

retval = dev->type->uevent(dev, env);

drivers/base/core.c

2731

struct kobj_uevent_env *env = NULL;

drivers/base/core.c

2752

env = kzalloc_obj(struct kobj_uevent_env);

drivers/base/core.c

2753

if (!env)

drivers/base/core.c

2757

retval = kset->uevent_ops->uevent(&dev->kobj, env);

drivers/base/core.c

2762

for (i = 0; i < env->envp_idx; i++)

drivers/base/core.c

2763

len += sysfs_emit_at(buf, len, "%s\n", env->envp[i]);

drivers/base/core.c

2765

kfree(env);

drivers/base/cpu.c

382

static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/cpu.c

387

add_uevent_var(env, "MODALIAS=%s", buf);

drivers/base/firmware_loader/sysfs.c

70

static int do_firmware_uevent(const struct fw_sysfs *fw_sysfs, struct kobj_uevent_env *env)

drivers/base/firmware_loader/sysfs.c

72

if (add_uevent_var(env, "FIRMWARE=%s", fw_sysfs->fw_priv->fw_name))

drivers/base/firmware_loader/sysfs.c

74

if (add_uevent_var(env, "TIMEOUT=%i", __firmware_loading_timeout()))

drivers/base/firmware_loader/sysfs.c

76

if (add_uevent_var(env, "ASYNC=%d", fw_sysfs->nowait))

drivers/base/firmware_loader/sysfs.c

82

static int firmware_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/firmware_loader/sysfs.c

89

err = do_firmware_uevent(fw_sysfs, env);

drivers/base/platform.c

1373

static int platform_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/base/platform.c

1379

rc = of_device_uevent_modalias(dev, env);

drivers/base/platform.c

1383

rc = acpi_device_uevent_modalias(dev, env);

drivers/base/platform.c

1387

add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX,

drivers/bcma/main.c

32

static int bcma_device_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/bcma/main.c

637

static int bcma_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bcma/main.c

641

return add_uevent_var(env,

drivers/bus/fsl-mc/fsl-mc-bus.c

128

static int fsl_mc_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/fsl-mc/fsl-mc-bus.c

132

if (add_uevent_var(env, "MODALIAS=fsl-mc:v%08Xd%s",

drivers/bus/mhi/ep/main.c

1671

static int mhi_ep_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mhi/ep/main.c

1675

return add_uevent_var(env, "MODALIAS=" MHI_EP_DEVICE_MODALIAS_FMT,

drivers/bus/mhi/host/init.c

1420

static int mhi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mhi/host/init.c

1424

return add_uevent_var(env, "MODALIAS=" MHI_DEVICE_MODALIAS_FMT,

drivers/bus/mips_cdmm.c

70

static int mips_cdmm_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/mips_cdmm.c

75

retval = add_uevent_var(env, "CDMM_CPU=%u", cdev->cpu);

drivers/bus/mips_cdmm.c

79

retval = add_uevent_var(env, "CDMM_TYPE=0x%02x", cdev->type);

drivers/bus/mips_cdmm.c

83

retval = add_uevent_var(env, "CDMM_REV=%u", cdev->rev);

drivers/bus/mips_cdmm.c

87

retval = add_uevent_var(env, "MODALIAS=mipscdmm:t%02X", cdev->type);

drivers/bus/sunxi-rsb.c

175

static int sunxi_rsb_device_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/bus/sunxi-rsb.c

177

return of_device_uevent_modalias(dev, env);

drivers/cxl/core/port.c

2244

static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/cxl/core/port.c

2246

return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,

drivers/dax/bus.c

33

static int dax_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/dax/bus.c

39

return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);

drivers/dma/idxd/bus.c

70

static int idxd_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/dma/idxd/bus.c

72

return add_uevent_var(env, "MODALIAS=" IDXD_DEVICES_MODALIAS_FMT, 0);

drivers/eisa/eisa-bus.c

130

static int eisa_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/eisa/eisa-bus.c

134

add_uevent_var(env, "MODALIAS=" EISA_DEVICE_MODALIAS_FMT, edev->id.sig);

drivers/firewire/core-device.c

242

static int fw_unit_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firewire/core-device.c

249

if (add_uevent_var(env, "MODALIAS=%s", modalias))

drivers/firmware/arm_ffa/bus.c

67

static int ffa_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/arm_ffa/bus.c

71

return add_uevent_var(env, "MODALIAS=" FFA_UEVENT_MODALIAS_FMT,

drivers/firmware/arm_scmi/bus.c

280

static int scmi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/arm_scmi/bus.c

284

return add_uevent_var(env, "MODALIAS=" SCMI_UEVENT_MODALIAS_FMT,

drivers/firmware/dmi-id.c

159

static int dmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/dmi-id.c

163

if (add_uevent_var(env, "MODALIAS="))

drivers/firmware/dmi-id.c

165

len = get_modalias(&env->buf[env->buflen - 1],

drivers/firmware/dmi-id.c

166

sizeof(env->buf) - env->buflen);

drivers/firmware/dmi-id.c

167

if (len >= (sizeof(env->buf) - env->buflen))

drivers/firmware/dmi-id.c

169

env->buflen += len;

drivers/firmware/google/coreboot_table.c

65

static int coreboot_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/firmware/google/coreboot_table.c

70

return add_uevent_var(env, "MODALIAS=coreboot:t%08X", tag);

drivers/fpga/dfl.c

285

static int dfl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/fpga/dfl.c

289

return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X",

drivers/gnss/core.c

367

static int gnss_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/gnss/core.c

372

ret = add_uevent_var(env, "GNSS_TYPE=%s", gnss_type_name(gdev));

drivers/gpu/drm/display/drm_dp_aux_bus.c

165

static int dp_aux_ep_dev_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/gpu/drm/display/drm_dp_aux_bus.c

167

return of_device_uevent_modalias(dev, env);

drivers/gpu/drm/drm_mipi_dsi.c

69

static int mipi_dsi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/gpu/drm/drm_mipi_dsi.c

74

err = of_device_uevent_modalias(dev, env);

drivers/gpu/drm/drm_mipi_dsi.c

78

add_uevent_var(env, "MODALIAS=%s%s", MIPI_DSI_MODULE_PREFIX,

drivers/gpu/drm/i915/gvt/handlers.c

1527

char *env[3] = {NULL, NULL, NULL};

drivers/gpu/drm/i915/gvt/handlers.c

1532

env[0] = display_ready_str;

drivers/gpu/drm/i915/gvt/handlers.c

1535

env[1] = vmid_str;

drivers/gpu/drm/i915/gvt/handlers.c

1537

return kobject_uevent_env(kobj, KOBJ_ADD, env);

drivers/gpu/host1x/bus.c

342

struct kobj_uevent_env *env)

drivers/gpu/host1x/bus.c

344

of_device_uevent(dev->parent, env);

drivers/greybus/core.c

111

static int greybus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/greybus/core.c

147

if (add_uevent_var(env, "BUS=%u", hd->bus_id))

drivers/greybus/core.c

151

if (add_uevent_var(env, "MODULE=%u", module->module_id))

drivers/greybus/core.c

156

if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))

drivers/greybus/core.c

158

if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",

drivers/greybus/core.c

169

if (add_uevent_var(env, "BUNDLE=%u", bundle->id))

drivers/greybus/core.c

171

if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))

drivers/hid/hid-core.c

2871

static int hid_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hid/hid-core.c

2875

if (add_uevent_var(env, "HID_ID=%04X:%08X:%08X",

drivers/hid/hid-core.c

2879

if (add_uevent_var(env, "HID_NAME=%s", hdev->name))

drivers/hid/hid-core.c

2882

if (add_uevent_var(env, "HID_PHYS=%s", hdev->phys))

drivers/hid/hid-core.c

2885

if (add_uevent_var(env, "HID_UNIQ=%s", hdev->uniq))

drivers/hid/hid-core.c

2888

if (add_uevent_var(env, "MODALIAS=hid:b%04Xg%04Xv%08Xp%08X",

drivers/hid/intel-ish-hid/ishtp/bus.c

372

static int ishtp_cl_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hid/intel-ish-hid/ishtp/bus.c

374

if (add_uevent_var(env, "MODALIAS=" ISHTP_MODULE_PREFIX "%s", dev_name(dev)))

drivers/hsi/hsi_core.c

33

static int hsi_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/hsi/hsi_core.c

35

add_uevent_var(env, "MODALIAS=hsi:%s", dev_name(dev));

drivers/hv/vmbus_drv.c

664

static int vmbus_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/hv/vmbus_drv.c

669

return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type);

drivers/hwmon/ultra45_env.c

102

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

121

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

144

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

167

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

188

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

204

struct env *p = dev_get_drvdata(dev);

drivers/hwmon/ultra45_env.c

257

struct env *p = devm_kzalloc(&op->dev, sizeof(*p), GFP_KERNEL);

drivers/hwmon/ultra45_env.c

296

struct env *p = platform_get_drvdata(op);

drivers/hwmon/ultra45_env.c

69

static u8 env_read(struct env *p, u8 ireg)

drivers/hwmon/ultra45_env.c

81

static void env_write(struct env *p, u8 ireg, u8 val)

drivers/i2c/i2c-core-base.c

162

static int i2c_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/i2c/i2c-core-base.c

167

rc = of_device_uevent_modalias(dev, env);

drivers/i2c/i2c-core-base.c

171

rc = acpi_device_uevent_modalias(dev, env);

drivers/i2c/i2c-core-base.c

175

return add_uevent_var(env, "MODALIAS=%s%s", I2C_MODULE_PREFIX, client->name);

drivers/i3c/master.c

312

static int i3c_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/i3c/master.c

325

return add_uevent_var(env, "MODALIAS=i3c:dcr%02Xmanuf%04X",

drivers/i3c/master.c

328

return add_uevent_var(env,

drivers/iio/adc/envelope-detector.c

101

synchronize_irq(env->comp_irq);

drivers/iio/adc/envelope-detector.c

104

spin_lock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

105

comp = env->comp;

drivers/iio/adc/envelope-detector.c

106

env->comp = 0;

drivers/iio/adc/envelope-detector.c

107

spin_unlock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

110

enable_irq(env->comp_irq);

drivers/iio/adc/envelope-detector.c

117

struct envelope *env = ctx;

drivers/iio/adc/envelope-detector.c

119

spin_lock(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

120

env->comp = 1;

drivers/iio/adc/envelope-detector.c

121

disable_irq_nosync(env->comp_irq);

drivers/iio/adc/envelope-detector.c

122

spin_unlock(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

127

static void envelope_detector_setup_compare(struct envelope *env)

drivers/iio/adc/envelope-detector.c

141

env->level = (env->high + env->low + !env->invert) / 2;

drivers/iio/adc/envelope-detector.c

143

if (env->high == env->low + 1) {

drivers/iio/adc/envelope-detector.c

144

complete(&env->done);

drivers/iio/adc/envelope-detector.c

149

ret = iio_write_channel_raw(env->dac, env->invert ? 0 : env->dac_max);

drivers/iio/adc/envelope-detector.c

154

envelope_detector_comp_latch(env);

drivers/iio/adc/envelope-detector.c

157

ret = iio_write_channel_raw(env->dac, env->level);

drivers/iio/adc/envelope-detector.c

162

schedule_delayed_work(&env->comp_timeout,

drivers/iio/adc/envelope-detector.c

163

msecs_to_jiffies(env->comp_interval));

drivers/iio/adc/envelope-detector.c

167

env->level = ret;

drivers/iio/adc/envelope-detector.c

168

complete(&env->done);

drivers/iio/adc/envelope-detector.c

173

struct envelope *env = container_of(work, struct envelope,

drivers/iio/adc/envelope-detector.c

177

if (!envelope_detector_comp_latch(env) ^ !env->invert)

drivers/iio/adc/envelope-detector.c

178

env->low = env->level;

drivers/iio/adc/envelope-detector.c

180

env->high = env->level;

drivers/iio/adc/envelope-detector.c

183

envelope_detector_setup_compare(env);

drivers/iio/adc/envelope-detector.c

190

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

205

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

206

env->high = env->dac_max + env->invert;

drivers/iio/adc/envelope-detector.c

207

env->low = -1 + env->invert;

drivers/iio/adc/envelope-detector.c

208

envelope_detector_setup_compare(env);

drivers/iio/adc/envelope-detector.c

209

wait_for_completion(&env->done);

drivers/iio/adc/envelope-detector.c

210

if (env->level < 0) {

drivers/iio/adc/envelope-detector.c

211

ret = env->level;

drivers/iio/adc/envelope-detector.c

214

*val = env->invert ? env->dac_max - env->level : env->level;

drivers/iio/adc/envelope-detector.c

215

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

220

return iio_read_channel_scale(env->dac, val, val2);

drivers/iio/adc/envelope-detector.c

226

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

234

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

236

return sprintf(buf, "%u\n", env->invert);

drivers/iio/adc/envelope-detector.c

244

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

255

trigger = invert ? env->comp_irq_trigger_inv : env->comp_irq_trigger;

drivers/iio/adc/envelope-detector.c

257

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

258

if (invert != env->invert)

drivers/iio/adc/envelope-detector.c

259

ret = irq_set_irq_type(env->comp_irq, trigger);

drivers/iio/adc/envelope-detector.c

261

env->invert = invert;

drivers/iio/adc/envelope-detector.c

264

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

274

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

276

return sprintf(buf, "%u\n", env->comp_interval);

drivers/iio/adc/envelope-detector.c

284

struct envelope *env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

294

mutex_lock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

295

env->comp_interval = interval;

drivers/iio/adc/envelope-detector.c

296

mutex_unlock(&env->read_lock);

drivers/iio/adc/envelope-detector.c

327

struct envelope *env;

drivers/iio/adc/envelope-detector.c

331

indio_dev = devm_iio_device_alloc(dev, sizeof(*env));

drivers/iio/adc/envelope-detector.c

336

env = iio_priv(indio_dev);

drivers/iio/adc/envelope-detector.c

337

env->comp_interval = 50; /* some sensible default? */

drivers/iio/adc/envelope-detector.c

339

spin_lock_init(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

340

mutex_init(&env->read_lock);

drivers/iio/adc/envelope-detector.c

341

init_completion(&env->done);

drivers/iio/adc/envelope-detector.c

342

INIT_DELAYED_WORK(&env->comp_timeout, envelope_detector_timeout);

drivers/iio/adc/envelope-detector.c

349

env->dac = devm_iio_channel_get(dev, "dac");

drivers/iio/adc/envelope-detector.c

350

if (IS_ERR(env->dac))

drivers/iio/adc/envelope-detector.c

351

return dev_err_probe(dev, PTR_ERR(env->dac),

drivers/iio/adc/envelope-detector.c

354

env->comp_irq = platform_get_irq_byname(pdev, "comp");

drivers/iio/adc/envelope-detector.c

355

if (env->comp_irq < 0)

drivers/iio/adc/envelope-detector.c

356

return env->comp_irq;

drivers/iio/adc/envelope-detector.c

358

ret = devm_request_irq(dev, env->comp_irq, envelope_detector_comp_isr,

drivers/iio/adc/envelope-detector.c

359

0, "envelope-detector", env);

drivers/iio/adc/envelope-detector.c

363

env->comp_irq_trigger = irq_get_trigger_type(env->comp_irq);

drivers/iio/adc/envelope-detector.c

364

if (env->comp_irq_trigger & IRQF_TRIGGER_RISING)

drivers/iio/adc/envelope-detector.c

365

env->comp_irq_trigger_inv |= IRQF_TRIGGER_FALLING;

drivers/iio/adc/envelope-detector.c

366

if (env->comp_irq_trigger & IRQF_TRIGGER_FALLING)

drivers/iio/adc/envelope-detector.c

367

env->comp_irq_trigger_inv |= IRQF_TRIGGER_RISING;

drivers/iio/adc/envelope-detector.c

368

if (env->comp_irq_trigger & IRQF_TRIGGER_HIGH)

drivers/iio/adc/envelope-detector.c

369

env->comp_irq_trigger_inv |= IRQF_TRIGGER_LOW;

drivers/iio/adc/envelope-detector.c

370

if (env->comp_irq_trigger & IRQF_TRIGGER_LOW)

drivers/iio/adc/envelope-detector.c

371

env->comp_irq_trigger_inv |= IRQF_TRIGGER_HIGH;

drivers/iio/adc/envelope-detector.c

373

ret = iio_get_channel_type(env->dac, &type);

drivers/iio/adc/envelope-detector.c

382

ret = iio_read_max_channel_raw(env->dac, &env->dac_max);

drivers/iio/adc/envelope-detector.c

79

static int envelope_detector_comp_latch(struct envelope *env)

drivers/iio/adc/envelope-detector.c

83

spin_lock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

84

comp = env->comp;

drivers/iio/adc/envelope-detector.c

85

env->comp = 0;

drivers/iio/adc/envelope-detector.c

86

spin_unlock_irq(&env->comp_lock);

drivers/iio/adc/envelope-detector.c

98

enable_irq(env->comp_irq);

drivers/infiniband/core/device.c

500

struct kobj_uevent_env *env)

drivers/infiniband/core/device.c

502

if (add_uevent_var(env, "NAME=%s", dev_name(device)))

drivers/input/input.c

1573

static int input_add_uevent_bm_var(struct kobj_uevent_env *env,

drivers/input/input.c

1578

if (add_uevent_var(env, "%s", name))

drivers/input/input.c

1581

len = input_print_bitmap(&env->buf[env->buflen - 1],

drivers/input/input.c

1582

sizeof(env->buf) - env->buflen,

drivers/input/input.c

1584

if (len >= (sizeof(env->buf) - env->buflen))

drivers/input/input.c

1587

env->buflen += len;

drivers/input/input.c

1608

static int input_add_uevent_modalias_var(struct kobj_uevent_env *env,

drivers/input/input.c

1613

if (add_uevent_var(env, "MODALIAS="))

drivers/input/input.c

1616

len = input_print_modalias(&env->buf[env->buflen - 1],

drivers/input/input.c

1617

(int)sizeof(env->buf) - env->buflen -

drivers/input/input.c

1620

if (len >= ((int)sizeof(env->buf) - env->buflen -

drivers/input/input.c

1624

env->buflen += len;

drivers/input/input.c

1630

int err = add_uevent_var(env, fmt, val); \

drivers/input/input.c

1637

int err = input_add_uevent_bm_var(env, name, bm, max); \

drivers/input/input.c

1644

int err = input_add_uevent_modalias_var(env, dev); \

drivers/input/input.c

1649

static int input_dev_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/input/serio/serio.c

865

int err = add_uevent_var(env, fmt, val); \

drivers/input/serio/serio.c

870

static int serio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ipack/ipack.c

79

static int ipack_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ipack/ipack.c

88

if (add_uevent_var(env,

drivers/isdn/mISDN/core.c

143

static int mISDN_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/isdn/mISDN/core.c

150

if (add_uevent_var(env, "nchans=%d", mdev->nrbchan))

drivers/macintosh/macio_asic.c

132

static int macio_device_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/macintosh/macio_asic.c

134

return of_device_uevent_modalias(dev, env);

drivers/mcb/mcb-core.c

44

static int mcb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mcb/mcb-core.c

49

ret = add_uevent_var(env, "MODALIAS=mcb:16z%03d", mdev->id);

drivers/md/bcache/super.c

1055

char *env[] = {

drivers/md/bcache/super.c

1093

kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);

drivers/md/bcache/super.c

1110

kfree(env[1]);

drivers/md/bcache/super.c

1111

kfree(env[2]);

drivers/md/bcache/sysfs.c

296

struct kobj_uevent_env *env;

drivers/md/bcache/sysfs.c

418

env = kzalloc_obj(struct kobj_uevent_env);

drivers/md/bcache/sysfs.c

419

if (!env)

drivers/md/bcache/sysfs.c

421

add_uevent_var(env, "DRIVER=bcache");

drivers/md/bcache/sysfs.c

422

add_uevent_var(env, "CACHED_UUID=%pU", dc->sb.uuid);

drivers/md/bcache/sysfs.c

423

add_uevent_var(env, "CACHED_LABEL=%s", buf);

drivers/md/bcache/sysfs.c

426

env->envp);

drivers/md/bcache/sysfs.c

427

kfree(env);

drivers/media/dvb-core/dvbdev.c

1066

static int dvb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/media/dvb-core/dvbdev.c

1070

add_uevent_var(env, "DVB_ADAPTER_NUM=%d", dvbdev->adapter->num);

drivers/media/dvb-core/dvbdev.c

1071

add_uevent_var(env, "DVB_DEVICE_TYPE=%s", dnames[dvbdev->type]);

drivers/media/dvb-core/dvbdev.c

1072

add_uevent_var(env, "DVB_DEVICE_NUM=%d", dvbdev->id);

drivers/media/rc/rc-main.c

1617

static int rc_dev_uevent(const struct device *device, struct kobj_uevent_env *env)

drivers/media/rc/rc-main.c

1627

ret = add_uevent_var(env, "NAME=%s", dev->rc_map.name);

drivers/media/rc/rc-main.c

1629

ret = add_uevent_var(env, "DRV_NAME=%s", dev->driver_name);

drivers/media/rc/rc-main.c

1631

ret = add_uevent_var(env, "DEV_NAME=%s", dev->device_name);

drivers/memstick/core/memstick.c

59

static int memstick_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/memstick/core/memstick.c

64

if (add_uevent_var(env, "MEMSTICK_TYPE=%02X", card->id.type))

drivers/memstick/core/memstick.c

67

if (add_uevent_var(env, "MEMSTICK_CATEGORY=%02X", card->id.category))

drivers/memstick/core/memstick.c

70

if (add_uevent_var(env, "MEMSTICK_CLASS=%02X", card->id.class))

drivers/misc/mei/bus.c

1268

static int mei_cl_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/misc/mei/bus.c

1274

if (add_uevent_var(env, "MEI_CL_VERSION=%d", version))

drivers/misc/mei/bus.c

1277

if (add_uevent_var(env, "MEI_CL_UUID=%pUl", uuid))

drivers/misc/mei/bus.c

1280

if (add_uevent_var(env, "MEI_CL_NAME=%s", cldev->name))

drivers/misc/mei/bus.c

1283

if (add_uevent_var(env, "MODALIAS=mei:%s:%pUl:%02X:",

drivers/misc/tifm_core.c

58

static int tifm_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/misc/tifm_core.c

62

if (add_uevent_var(env, "TIFM_CARD_TYPE=%s", tifm_media_type_name(sock->type, 1)))

drivers/mmc/core/bus.c

101

retval = add_uevent_var(env, "SDIO_INFO%u=%s", i+1, card->info[i]);

drivers/mmc/core/bus.c

114

retval = add_uevent_var(env, "MMC_NAME=%s", mmc_card_name(card));

drivers/mmc/core/bus.c

122

retval = add_uevent_var(env, "MODALIAS=mmc:block");

drivers/mmc/core/bus.c

59

mmc_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mmc/core/bus.c

84

retval = add_uevent_var(env, "MMC_TYPE=%s", type);

drivers/mmc/core/bus.c

90

retval = add_uevent_var(env, "SDIO_ID=%04X:%04X",

drivers/mmc/core/bus.c

95

retval = add_uevent_var(env, "SDIO_REVISION=%u.%u",

drivers/mmc/core/sdio_bus.c

123

sdio_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/mmc/core/sdio_bus.c

128

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

132

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

136

if (add_uevent_var(env,

drivers/mmc/core/sdio_bus.c

141

if (add_uevent_var(env, "SDIO_INFO%u=%s", i+1, func->info[i]))

drivers/mmc/core/sdio_bus.c

145

if (add_uevent_var(env,

drivers/net/ethernet/mellanox/mlxsw/core.c

2195

err = mlxsw_env_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

2210

mlxsw_env_fini(mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

2292

mlxsw_env_fini(mlxsw_core->env);

drivers/net/ethernet/mellanox/mlxsw/core.c

3211

return mlxsw_core->env;

drivers/net/ethernet/mellanox/mlxsw/core.c

98

struct mlxsw_env *env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1240

static int mlxsw_env_line_cards_alloc(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1245

for (i = 0; i < env->num_of_slots; i++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1246

env->line_cards[i] = kzalloc_flex(*env->line_cards[i],

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1248

env->max_module_count);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1249

if (!env->line_cards[i])

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1255

for (j = 0; j < env->max_module_count; j++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1256

module_info = &env->line_cards[i]->module_info[j];

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1266

kfree(env->line_cards[i]);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1270

static void mlxsw_env_line_cards_free(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1272

int i = env->num_of_slots;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1275

kfree(env->line_cards[i]);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1326

struct mlxsw_env *env,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1331

for (i = 0; i < env->line_cards[slot_index]->module_count; i++) {

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1337

module_info = &env->line_cards[slot_index]->module_info[i];

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1343

dev_err(env->bus_info->dev, "%s\n", extack._msg);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1438

struct mlxsw_env *env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1455

env = kzalloc_flex(*env, line_cards, num_of_slots + 1);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1456

if (!env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1459

env->core = mlxsw_core;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1460

env->bus_info = bus_info;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1461

env->num_of_slots = num_of_slots + 1;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1462

env->max_module_count = max_module_count;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1463

err = mlxsw_env_line_cards_alloc(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1467

mutex_init(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1468

*p_env = env;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1470

err = mlxsw_linecards_event_ops_register(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1471

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1486

env->line_cards[0]->module_count = num_of_slots ? 0 : module_count;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1491

err = mlxsw_env_module_event_enable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1499

mlxsw_env_max_module_eeprom_len_query(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1500

env->line_cards[0]->active = true;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1505

mlxsw_env_module_event_disable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1507

mlxsw_env_module_plug_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1509

mlxsw_env_temp_warn_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1511

mlxsw_linecards_event_ops_unregister(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1512

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1514

mutex_destroy(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1515

mlxsw_env_line_cards_free(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1517

kfree(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1521

void mlxsw_env_fini(struct mlxsw_env *env)

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1523

env->line_cards[0]->active = false;

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1524

mlxsw_env_module_event_disable(env, 0);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1525

mlxsw_env_module_plug_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1528

mlxsw_env_temp_warn_event_unregister(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1529

mlxsw_linecards_event_ops_unregister(env->core,

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1530

&mlxsw_env_event_ops, env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1531

mutex_destroy(&env->line_cards_lock);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1532

mlxsw_env_line_cards_free(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.c

1533

kfree(env);

drivers/net/ethernet/mellanox/mlxsw/core_env.h

72

void mlxsw_env_fini(struct mlxsw_env *env);

drivers/net/ethernet/netronome/nfp/bpf/main.h

566

int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx,

drivers/net/ethernet/netronome/nfp/bpf/main.h

568

int nfp_bpf_finalize(struct bpf_verifier_env *env);

drivers/net/ethernet/netronome/nfp/bpf/main.h

570

int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

drivers/net/ethernet/netronome/nfp/bpf/main.h

572

int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

113

pr_vlog(env, "value at offset %d/%d may be non-zero, bpf_map_update_elem() is required to initialize atomic counters to zero to avoid offload endian issues\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

124

nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

131

pr_vlog(env, "%s: unsupported ptr type %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

136

pr_vlog(env, "%s: variable pointer\n", fname);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

142

pr_vlog(env, "%s: unaligned stack pointer %lld\n", fname, -off);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

157

nfp_bpf_map_call_ok(const char *fname, struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

16

#define pr_vlog(env, fmt, ...) \

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

162

pr_vlog(env, "%s: not supported by FW\n", fname);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

17

bpf_verifier_log_write(env, "[nfp] " fmt, ##__VA_ARGS__)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

171

struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

174

const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

175

const struct bpf_reg_state *reg2 = cur_regs(env) + BPF_REG_2;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

176

const struct bpf_reg_state *reg3 = cur_regs(env) + BPF_REG_3;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

183

pr_vlog(env, "adjust_head not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

187

pr_vlog(env, "adjust_head: FW requires shifting metadata, not supported by the driver\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

196

pr_vlog(env, "adjust_tail not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

202

if (!nfp_bpf_map_call_ok("map_lookup", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

204

!nfp_bpf_stack_arg_ok("map_lookup", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

210

if (!nfp_bpf_map_call_ok("map_update", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

212

!nfp_bpf_stack_arg_ok("map_update", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

214

!nfp_bpf_stack_arg_ok("map_update", env, reg3, NULL) ||

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

215

!nfp_bpf_map_update_value_ok(env))

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

220

if (!nfp_bpf_map_call_ok("map_delete", env, meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

222

!nfp_bpf_stack_arg_ok("map_delete", env, reg2,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

230

pr_vlog(env, "bpf_get_prandom_u32(): FW doesn't support random number generation\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

240

pr_vlog(env, "event_output: not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

253

pr_vlog(env, "event_output: must use BPF_F_CURRENT_CPU, var_off: %s\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

261

reg1 = cur_regs(env) + BPF_REG_4;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

267

pr_vlog(env, "event_output: unsupported ptr type: %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

273

!nfp_bpf_stack_arg_ok("event_output", env, reg1, NULL))

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

286

pr_vlog(env, "warning: return codes and behavior of event_output helper differ for offload!\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

292

pr_vlog(env, "event_output: ptr type changed: %d %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

299

pr_vlog(env, "unsupported function id: %d\n", func_id);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

312

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

314

const struct bpf_reg_state *reg0 = cur_regs(env) + BPF_REG_0;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

324

pr_vlog(env, "unsupported exit state: %d, var_off: %s\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

334

pr_vlog(env, "unsupported exit state: %d, imm: %llx\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

346

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

350

if (reg->frameno != env->cur_state->curframe)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

354

pr_vlog(env, "variable ptr stack access\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

372

pr_vlog(env, "stack access changed location was:%d is:%d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

392

nfp_bpf_map_mark_used_one(struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

398

pr_vlog(env, "map value use type conflict %s vs %s off: %u\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

406

pr_vlog(env, "atomic counter in map value may already be initialized to non-zero value off: %u\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

417

nfp_bpf_map_mark_used(struct bpf_verifier_env *env, struct nfp_insn_meta *meta,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

427

pr_vlog(env, "map value offset is variable\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

437

pr_vlog(env, "map value access out-of-bounds\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

442

err = nfp_bpf_map_mark_used_one(env, nfp_map, off + i, use);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

452

struct bpf_verifier_env *env, u8 reg_no)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

454

const struct bpf_reg_state *reg = cur_regs(env) + reg_no;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

461

pr_vlog(env, "unsupported ptr type: %d\n", reg->type);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

466

err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

473

err = nfp_bpf_map_mark_used(env, meta, reg,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

479

pr_vlog(env, "map writes not supported\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

483

err = nfp_bpf_map_mark_used(env, meta, reg,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

491

pr_vlog(env, "ptr type changed for instruction %d -> %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

503

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

505

const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

514

pr_vlog(env, "queue selection not supported by FW\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

518

pr_vlog(env, "unsupported store to context field\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

522

return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

527

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

529

const struct bpf_reg_state *sreg = cur_regs(env) + meta->insn.src_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

530

const struct bpf_reg_state *dreg = cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

533

pr_vlog(env, "atomic op not implemented: %d\n", meta->insn.imm);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

538

pr_vlog(env, "atomic add not to a map value pointer: %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

543

pr_vlog(env, "atomic add not of a scalar: %d\n", sreg->type);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

552

return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

557

struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

560

cur_regs(env) + meta->insn.src_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

562

cur_regs(env) + meta->insn.dst_reg;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

583

pr_vlog(env, "multiplier is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

587

pr_vlog(env, "multiplicand is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

592

pr_vlog(env, "sign extended multiplicand won't be within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

609

pr_vlog(env, "dividend is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

614

pr_vlog(env, "divisor is not constant\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

618

pr_vlog(env, "divisor is not within u32 value range\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

623

pr_vlog(env, "divide by negative constant is not supported\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

631

int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

634

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

641

pr_vlog(env, "instruction %#02x not supported\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

648

pr_vlog(env, "program uses extended registers - jit hardening?\n");

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

653

return nfp_bpf_check_helper_call(nfp_prog, env, meta);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

655

return nfp_bpf_check_exit(nfp_prog, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

658

return nfp_bpf_check_ptr(nfp_prog, meta, env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

661

return nfp_bpf_check_store(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

664

return nfp_bpf_check_atomic(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

667

return nfp_bpf_check_alu(nfp_prog, meta, env);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

673

nfp_assign_subprog_idx_and_regs(struct bpf_verifier_env *env,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

690

pr_vlog(env, "BUG: number of processed BPF functions is not consistent (processed %d, expected %d)\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

763

int nfp_bpf_finalize(struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

771

nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

772

nfp_prog->subprog_cnt = env->subprog_cnt;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

778

nfp_assign_subprog_idx_and_regs(env, nfp_prog);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

780

info = env->subprog_info;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

794

nn = netdev_priv(env->prog->aux->offload->netdev);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

798

pr_vlog(env, "stack too large: program %dB > FW stack %dB\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

803

nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

807

int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

810

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

811

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

828

pr_vlog(env, "branch hard wire at %d changes target %d -> %d\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

83

static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

836

pr_vlog(env, "unsupported instruction replacement %hhx -> %hhx\n",

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

841

int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

843

struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

844

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

85

const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

86

const struct bpf_reg_state *reg3 = cur_regs(env) + BPF_REG_3;

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

92

state = env->cur_state->frame[reg3->frameno];

drivers/net/netdevsim/bpf.c

26

#define pr_vlog(env, fmt, ...) \

drivers/net/netdevsim/bpf.c

27

bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)

drivers/net/netdevsim/bpf.c

63

nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)

drivers/net/netdevsim/bpf.c

68

state = env->prog->aux->offload->dev_priv;

drivers/net/netdevsim/bpf.c

72

if (insn_idx == env->prog->len - 1) {

drivers/net/netdevsim/bpf.c

73

pr_vlog(env, "Hello from netdevsim!\n");

drivers/net/netdevsim/bpf.c

82

static int nsim_bpf_finalize(struct bpf_verifier_env *env)

drivers/net/phy/mdio_bus.c

977

static int mdio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/net/phy/mdio_bus.c

982

rc = of_device_uevent_modalias(dev, env);

drivers/net/wireless/mediatek/mt76/mt7921/mcu.c

1321

u8 env;

drivers/net/wireless/mediatek/mt76/mt7921/mcu.c

1332

.env = env_cap,

drivers/net/wireless/mediatek/mt76/mt7925/mcu.c

3370

u8 env;

drivers/net/wireless/mediatek/mt76/mt7925/mcu.c

3381

.env = env_cap,

drivers/net/wireless/realtek/rtw89/mac80211.c

1923

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/mac80211.c

1926

rpt = list_first_entry_or_null(&env->nhm_rpt_list, typeof(*rpt), list);

drivers/net/wireless/realtek/rtw89/phy.c

5786

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5788

return time_us >> (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5794

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5796

return idx << (ilog2(CCX_US_BASE_RATIO) + env->ccx_unit_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5803

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5806

env->ccx_manual_ctrl = false;

drivers/net/wireless/realtek/rtw89/phy.c

5807

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

5808

env->ccx_rac_lv = RTW89_RAC_RELEASE;

drivers/net/wireless/realtek/rtw89/phy.c

5809

env->ccx_period = 0;

drivers/net/wireless/realtek/rtw89/phy.c

5810

env->ccx_unit_idx = RTW89_CCX_32_US;

drivers/net/wireless/realtek/rtw89/phy.c

5825

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5829

numer = report * score + (env->ccx_period >> 1);

drivers/net/wireless/realtek/rtw89/phy.c

5830

if (env->ccx_period)

drivers/net/wireless/realtek/rtw89/phy.c

5831

ret = numer / env->ccx_period;

drivers/net/wireless/realtek/rtw89/phy.c

5868

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5871

"lv:(%d)->(0)\n", env->ccx_rac_lv);

drivers/net/wireless/realtek/rtw89/phy.c

5873

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

5874

env->ccx_rac_lv = RTW89_RAC_RELEASE;

drivers/net/wireless/realtek/rtw89/phy.c

5875

env->ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;

drivers/net/wireless/realtek/rtw89/phy.c

5882

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5883

bool is_update = env->ifs_clm_app != para->ifs_clm_app;

drivers/net/wireless/realtek/rtw89/phy.c

5885

u16 *ifs_th_l = env->ifs_clm_th_l;

drivers/net/wireless/realtek/rtw89/phy.c

5886

u16 *ifs_th_h = env->ifs_clm_th_h;

drivers/net/wireless/realtek/rtw89/phy.c

5937

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5942

env->ifs_clm_th_l[0], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5944

env->ifs_clm_th_l[1], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5946

env->ifs_clm_th_l[2], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5948

env->ifs_clm_th_l[3], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5951

env->ifs_clm_th_h[0], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5953

env->ifs_clm_th_h[1], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5955

env->ifs_clm_th_h[2], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5957

env->ifs_clm_th_h[3], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

5962

i + 1, env->ifs_clm_th_l[i], env->ifs_clm_th_h[i]);

drivers/net/wireless/realtek/rtw89/phy.c

5969

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

5972

env->nhm_include_cca = false;

drivers/net/wireless/realtek/rtw89/phy.c

5973

env->nhm_mntr_time = 0;

drivers/net/wireless/realtek/rtw89/phy.c

5974

env->nhm_sum = 0;

drivers/net/wireless/realtek/rtw89/phy.c

5997

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6001

env->ifs_clm_app = RTW89_IFS_CLM_BACKGROUND;

drivers/net/wireless/realtek/rtw89/phy.c

6002

env->ifs_clm_mntr_time = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6024

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6034

"ccx_ongoing=%d, level:(%d)->(%d)\n", env->ccx_ongoing,

drivers/net/wireless/realtek/rtw89/phy.c

6035

env->ccx_rac_lv, level);

drivers/net/wireless/realtek/rtw89/phy.c

6037

if (env->ccx_ongoing) {

drivers/net/wireless/realtek/rtw89/phy.c

6038

if (level <= env->ccx_rac_lv)

drivers/net/wireless/realtek/rtw89/phy.c

6041

env->ccx_ongoing = false;

drivers/net/wireless/realtek/rtw89/phy.c

6045

env->ccx_rac_lv = level;

drivers/net/wireless/realtek/rtw89/phy.c

6057

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6076

env->ccx_ongoing = true;

drivers/net/wireless/realtek/rtw89/phy.c

6082

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6086

env->ifs_clm_tx_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6087

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_tx, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6088

env->ifs_clm_edcca_excl_cca_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6089

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_edcca_excl_cca,

drivers/net/wireless/realtek/rtw89/phy.c

6091

env->ifs_clm_cck_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6092

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6093

env->ifs_clm_ofdm_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6094

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERCENT);

drivers/net/wireless/realtek/rtw89/phy.c

6095

env->ifs_clm_cck_cca_excl_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6096

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckcca_excl_fa,

drivers/net/wireless/realtek/rtw89/phy.c

6098

env->ifs_clm_ofdm_cca_excl_fa_ratio =

drivers/net/wireless/realtek/rtw89/phy.c

6099

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmcca_excl_fa,

drivers/net/wireless/realtek/rtw89/phy.c

6101

env->ifs_clm_cck_fa_permil =

drivers/net/wireless/realtek/rtw89/phy.c

6102

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_cckfa, PERMIL);

drivers/net/wireless/realtek/rtw89/phy.c

6103

env->ifs_clm_ofdm_fa_permil =

drivers/net/wireless/realtek/rtw89/phy.c

6104

rtw89_phy_ccx_get_report(rtwdev, bb, env->ifs_clm_ofdmfa, PERMIL);

drivers/net/wireless/realtek/rtw89/phy.c

6107

if (env->ifs_clm_his[i] > ENV_MNTR_IFSCLM_HIS_MAX) {

drivers/net/wireless/realtek/rtw89/phy.c

6108

env->ifs_clm_ifs_avg[i] = ENV_MNTR_FAIL_DWORD;

drivers/net/wireless/realtek/rtw89/phy.c

6110

env->ifs_clm_ifs_avg[i] =

drivers/net/wireless/realtek/rtw89/phy.c

6112

env->ifs_clm_avg[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6115

res = rtw89_phy_ccx_idx_to_us(rtwdev, bb, env->ifs_clm_cca[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6116

res += env->ifs_clm_his[i] >> 1;

drivers/net/wireless/realtek/rtw89/phy.c

6117

if (env->ifs_clm_his[i])

drivers/net/wireless/realtek/rtw89/phy.c

6118

res /= env->ifs_clm_his[i];

drivers/net/wireless/realtek/rtw89/phy.c

6121

env->ifs_clm_cca_avg[i] = res;

drivers/net/wireless/realtek/rtw89/phy.c

6126

env->ifs_clm_tx_ratio, env->ifs_clm_edcca_excl_cca_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6129

env->ifs_clm_cck_fa_ratio, env->ifs_clm_ofdm_fa_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6132

env->ifs_clm_cck_fa_permil, env->ifs_clm_ofdm_fa_permil);

drivers/net/wireless/realtek/rtw89/phy.c

6135

env->ifs_clm_cck_cca_excl_fa_ratio,

drivers/net/wireless/realtek/rtw89/phy.c

6136

env->ifs_clm_ofdm_cca_excl_fa_ratio);

drivers/net/wireless/realtek/rtw89/phy.c

6141

i + 1, env->ifs_clm_his[i], env->ifs_clm_ifs_avg[i],

drivers/net/wireless/realtek/rtw89/phy.c

6142

env->ifs_clm_cca_avg[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6147

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6153

if (env->nhm_sum == 0)

drivers/net/wireless/realtek/rtw89/phy.c

6156

weight_zero = clamp_t(u16, env->nhm_th[0] - RTW89_NHM_WEIGHT_OFFSET, 0, U8_MAX);

drivers/net/wireless/realtek/rtw89/phy.c

6162

nhm_weight[i] = env->nhm_th[i - 1] + RTW89_NHM_WEIGHT_OFFSET;

drivers/net/wireless/realtek/rtw89/phy.c

6164

nhm_weight[i] = (env->nhm_th[i - 1] + env->nhm_th[i]) / 2;

drivers/net/wireless/realtek/rtw89/phy.c

6169

if (env->nhm_th[RTW89_NHM_TH_NUM - 1] == RTW89_NHM_WA_TH) {

drivers/net/wireless/realtek/rtw89/phy.c

6171

env->nhm_th[RTW89_NHM_TH_NUM - 2] +

drivers/net/wireless/realtek/rtw89/phy.c

6177

env->nhm_result[0] += env->nhm_result[RTW89_NHM_RPT_NUM - 1];

drivers/net/wireless/realtek/rtw89/phy.c

6178

env->nhm_result[RTW89_NHM_RPT_NUM - 1] = 0;

drivers/net/wireless/realtek/rtw89/phy.c

6182

nhm_weighted_sum += env->nhm_result[i] * nhm_weight[i];

drivers/net/wireless/realtek/rtw89/phy.c

6184

return (nhm_weighted_sum / env->nhm_sum) >> RTW89_NHM_TH_FACTOR;

drivers/net/wireless/realtek/rtw89/phy.c

6192

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6211

env->nhm_result[i] =

drivers/net/wireless/realtek/rtw89/phy.c

6214

sum += env->nhm_result[i];

drivers/net/wireless/realtek/rtw89/phy.c

6216

env->nhm_sum = sum;

drivers/net/wireless/realtek/rtw89/phy.c

6236

rpt = &env->nhm_his[hw_band][chan_idx];

drivers/net/wireless/realtek/rtw89/phy.c

6237

nhm_list = &env->nhm_rpt_list;

drivers/net/wireless/realtek/rtw89/phy.c

6268

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6279

env->ifs_clm_tx =

drivers/net/wireless/realtek/rtw89/phy.c

6282

env->ifs_clm_edcca_excl_cca =

drivers/net/wireless/realtek/rtw89/phy.c

6285

env->ifs_clm_cckcca_excl_fa =

drivers/net/wireless/realtek/rtw89/phy.c

6288

env->ifs_clm_ofdmcca_excl_fa =

drivers/net/wireless/realtek/rtw89/phy.c

6291

env->ifs_clm_cckfa =

drivers/net/wireless/realtek/rtw89/phy.c

6294

env->ifs_clm_ofdmfa =

drivers/net/wireless/realtek/rtw89/phy.c

6298

env->ifs_clm_his[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6301

env->ifs_clm_his[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6305

env->ifs_clm_his[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6308

env->ifs_clm_his[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6312

env->ifs_clm_avg[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6315

env->ifs_clm_avg[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6318

env->ifs_clm_avg[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6321

env->ifs_clm_avg[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6325

env->ifs_clm_cca[0] =

drivers/net/wireless/realtek/rtw89/phy.c

6328

env->ifs_clm_cca[1] =

drivers/net/wireless/realtek/rtw89/phy.c

6331

env->ifs_clm_cca[2] =

drivers/net/wireless/realtek/rtw89/phy.c

6334

env->ifs_clm_cca[3] =

drivers/net/wireless/realtek/rtw89/phy.c

6338

env->ifs_clm_total_ifs =

drivers/net/wireless/realtek/rtw89/phy.c

6343

env->ifs_clm_total_ifs);

drivers/net/wireless/realtek/rtw89/phy.c

6346

env->ifs_clm_tx, env->ifs_clm_edcca_excl_cca);

drivers/net/wireless/realtek/rtw89/phy.c

6349

env->ifs_clm_cckfa, env->ifs_clm_ofdmfa);

drivers/net/wireless/realtek/rtw89/phy.c

6352

env->ifs_clm_cckcca_excl_fa, env->ifs_clm_ofdmcca_excl_fa);

drivers/net/wireless/realtek/rtw89/phy.c

6357

"T%d:[%d, %d, %d]\n", i + 1, env->ifs_clm_his[i],

drivers/net/wireless/realtek/rtw89/phy.c

6358

env->ifs_clm_avg[i], env->ifs_clm_cca[i]);

drivers/net/wireless/realtek/rtw89/phy.c

6368

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6377

env->nhm_th[i] = nhm_th_11k[i] << RTW89_NHM_TH_FACTOR;

drivers/net/wireless/realtek/rtw89/phy.c

6381

env->nhm_th[RTW89_NHM_TH_NUM - 1] = RTW89_NHM_WA_TH;

drivers/net/wireless/realtek/rtw89/phy.c

6387

env->nhm_th[i], bb->phy_idx);

drivers/net/wireless/realtek/rtw89/phy.c

6396

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6414

if (para->mntr_time != env->nhm_mntr_time) {

drivers/net/wireless/realtek/rtw89/phy.c

6422

env->nhm_mntr_time = para->mntr_time;

drivers/net/wireless/realtek/rtw89/phy.c

6423

env->ccx_period = period;

drivers/net/wireless/realtek/rtw89/phy.c

6424

env->ccx_unit_idx = unit_idx;

drivers/net/wireless/realtek/rtw89/phy.c

6427

if (para->nhm_incld_cca != env->nhm_include_cca) {

drivers/net/wireless/realtek/rtw89/phy.c

6432

env->nhm_include_cca = para->nhm_incld_cca;

drivers/net/wireless/realtek/rtw89/phy.c

6471

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6485

if (para->mntr_time != env->ifs_clm_mntr_time) {

drivers/net/wireless/realtek/rtw89/phy.c

6496

env->ifs_clm_mntr_time, para->mntr_time);

drivers/net/wireless/realtek/rtw89/phy.c

6498

env->ifs_clm_mntr_time = para->mntr_time;

drivers/net/wireless/realtek/rtw89/phy.c

6499

env->ccx_period = (u16)period;

drivers/net/wireless/realtek/rtw89/phy.c

6500

env->ccx_unit_idx = (u8)unit_idx;

drivers/net/wireless/realtek/rtw89/phy.c

6504

env->ifs_clm_app = para->ifs_clm_app;

drivers/net/wireless/realtek/rtw89/phy.c

6514

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6518

env->ccx_watchdog_result = RTW89_PHY_ENV_MON_CCX_FAIL;

drivers/net/wireless/realtek/rtw89/phy.c

6519

if (env->ccx_manual_ctrl) {

drivers/net/wireless/realtek/rtw89/phy.c

6530

env->ccx_watchdog_result |= RTW89_PHY_ENV_MON_IFS_CLM;

drivers/net/wireless/realtek/rtw89/phy.c

6544

env->ccx_watchdog_result, chk_result);

drivers/net/wireless/realtek/rtw89/phy.c

6932

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

6937

fa_ratio = env->ifs_clm_cck_fa_permil + env->ifs_clm_ofdm_fa_permil;

drivers/net/wireless/realtek/rtw89/phy.c

6964

env->ifs_clm_cck_fa_permil, env->ifs_clm_ofdm_fa_permil,

drivers/net/wireless/realtek/rtw89/phy.c

6965

env->ifs_clm_cck_fa_permil + env->ifs_clm_ofdm_fa_permil,

drivers/net/wireless/realtek/rtw89/phy.c

7679

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/phy.c

7695

env->nhm_his[hw_band] =

drivers/net/wireless/realtek/rtw89/phy.c

7697

sizeof(*env->nhm_his[0]), GFP_KERNEL);

drivers/net/wireless/realtek/rtw89/phy.c

7700

INIT_LIST_HEAD(&env->nhm_his[hw_band][idx].list);

drivers/net/wireless/realtek/rtw89/phy.c

7702

INIT_LIST_HEAD(&env->nhm_rpt_list);

drivers/net/wireless/realtek/rtw89/sar.c

631

struct rtw89_env_monitor_info *env = &bb->env_monitor;

drivers/net/wireless/realtek/rtw89/sar.c

633

u8 tx_ratio = env->ifs_clm_tx_ratio;

drivers/net/xen-netback/xenbus.c

204

struct kobj_uevent_env *env)

drivers/net/xen-netback/xenbus.c

211

if (add_uevent_var(env, "script=%s", be->hotplug_script))

drivers/net/xen-netback/xenbus.c

217

return add_uevent_var(env, "vif=%s", be->vif->dev->name);

drivers/nvdimm/bus.c

47

static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/nvdimm/bus.c

49

return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,

drivers/nvme/host/core.c

134

static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/nvme/host/core.c

4590

static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/nvme/host/core.c

4597

ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);

drivers/nvme/host/core.c

4602

ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);

drivers/nvme/host/core.c

4606

ret = add_uevent_var(env, "NVME_TRSVCID=%s",

drivers/nvme/host/core.c

4611

ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",

drivers/nvme/host/core.c

4616

ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",

drivers/nvmem/layouts.c

49

struct kobj_uevent_env *env)

drivers/nvmem/layouts.c

53

ret = of_device_uevent_modalias(dev, env);

drivers/of/device.c

215

void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/of/device.c

225

add_uevent_var(env, "OF_NAME=%pOFn", dev->of_node);

drivers/of/device.c

226

add_uevent_var(env, "OF_FULLNAME=%pOF", dev->of_node);

drivers/of/device.c

229

add_uevent_var(env, "OF_TYPE=%s", type);

drivers/of/device.c

235

add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);

drivers/of/device.c

238

add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);

drivers/of/device.c

244

add_uevent_var(env, "OF_ALIAS_%d=%s", seen,

drivers/of/device.c

253

int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)

drivers/of/device.c

261

if (add_uevent_var(env, "MODALIAS="))

drivers/of/device.c

264

sl = of_modalias(dev->of_node, &env->buf[env->buflen-1],

drivers/of/device.c

265

sizeof(env->buf) - env->buflen);

drivers/of/device.c

268

if (sl >= (sizeof(env->buf) - env->buflen))

drivers/of/device.c

270

env->buflen += sl;

drivers/pci/pci-driver.c

1584

static int pci_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pci/pci-driver.c

1593

if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))

drivers/pci/pci-driver.c

1596

if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))

drivers/pci/pci-driver.c

1599

if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,

drivers/pci/pci-driver.c

1603

if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))

drivers/pci/pci-driver.c

1606

if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",

drivers/pcmcia/cs.c

798

struct kobj_uevent_env *env)

drivers/pcmcia/cs.c

802

if (add_uevent_var(env, "SOCKET_NO=%u", s->sock))

drivers/pcmcia/ds.c

934

static int pcmcia_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pcmcia/ds.c

952

if (add_uevent_var(env, "SOCKET_NO=%u", p_dev->socket->sock))

drivers/pcmcia/ds.c

955

if (add_uevent_var(env, "DEVICE_NO=%02X", p_dev->device_no))

drivers/pcmcia/ds.c

958

if (add_uevent_var(env, "MODALIAS=pcmcia:m%04Xc%04Xf%02Xfn%02Xpfn%02X"

drivers/platform/chrome/cros_ec_lpc.c

446

static const char *env[] = { "ERROR=PANIC", NULL };

drivers/platform/chrome/cros_ec_lpc.c

456

kobject_uevent_env(&ec_dev->dev->kobj, KOBJ_CHANGE, (char **)env);

drivers/platform/raspberrypi/vchiq-interface/vchiq_bus.c

26

static int vchiq_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/raspberrypi/vchiq-interface/vchiq_bus.c

30

return add_uevent_var(env, "MODALIAS=vchiq:%s", dev_name(&device->dev));

drivers/platform/surface/aggregator/bus.c

41

static int ssam_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/surface/aggregator/bus.c

45

return add_uevent_var(env, "MODALIAS=ssam:d%02Xc%02Xt%02Xi%02Xf%02X",

drivers/platform/wmi/core.c

929

static int wmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/platform/wmi/core.c

933

if (add_uevent_var(env, "MODALIAS=wmi:%pUL", &wblock->gblock.guid))

drivers/platform/wmi/core.c

936

if (add_uevent_var(env, "WMI_GUID=%pUL", &wblock->gblock.guid))

drivers/platform/x86/asus-wmi.c

1811

static int kbd_led_read(struct asus_wmi *asus, int *level, int *env)

drivers/platform/x86/asus-wmi.c

1833

if (env)

drivers/platform/x86/asus-wmi.c

1834

*env = (retval >> 8) & 0x7F;

drivers/pnp/driver.c

153

static int pnp_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/pnp/driver.c

163

if (add_uevent_var(env, "MODALIAS=pnp:d%s", pos->id))

drivers/power/supply/power_supply.h

42

extern int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/power/supply/power_supply_sysfs.c

528

static int add_prop_uevent(const struct device *dev, struct kobj_uevent_env *env,

drivers/power/supply/power_supply_sysfs.c

555

return add_uevent_var(env, "POWER_SUPPLY_%s=%s",

drivers/power/supply/power_supply_sysfs.c

559

int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/power/supply/power_supply_sysfs.c

570

ret = add_uevent_var(env, "POWER_SUPPLY_NAME=%s", psy->desc->name);

drivers/power/supply/power_supply_sysfs.c

585

ret = add_prop_uevent(dev, env, POWER_SUPPLY_PROP_TYPE, prop_buf);

drivers/power/supply/power_supply_sysfs.c

590

ret = add_prop_uevent(dev, env, j, prop_buf);

drivers/rapidio/rio-driver.c

207

static int rio_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/rapidio/rio-driver.c

218

if (add_uevent_var(env, "MODALIAS=rapidio:v%04Xd%04Xav%04Xad%04X",

drivers/rpmsg/rpmsg_core.c

441

static int rpmsg_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/rpmsg/rpmsg_core.c

446

ret = of_device_uevent_modalias(dev, env);

drivers/rpmsg/rpmsg_core.c

450

return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT,

drivers/s390/cio/css.c

1404

static int css_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/css.c

1409

ret = add_uevent_var(env, "ST=%01X", sch->st);

drivers/s390/cio/css.c

1412

ret = add_uevent_var(env, "MODALIAS=css:t%01X", sch->st);

drivers/s390/cio/device.c

103

static int ccw_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/device.c

111

ret = add_uevent_var(env, "CU_TYPE=%04X", id->cu_type);

drivers/s390/cio/device.c

116

ret = add_uevent_var(env, "CU_MODEL=%02X", id->cu_model);

drivers/s390/cio/device.c

122

ret = add_uevent_var(env, "DEV_TYPE=%04X", id->dev_type);

drivers/s390/cio/device.c

127

ret = add_uevent_var(env, "DEV_MODEL=%02X", id->dev_model);

drivers/s390/cio/device.c

133

ret = add_uevent_var(env, "MODALIAS=%s", modalias_buf);

drivers/s390/cio/scm.c

41

static int scmdev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/cio/scm.c

43

return add_uevent_var(env, "MODALIAS=scm:scmdev");

drivers/s390/crypto/ap_bus.c

661

static int ap_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/s390/crypto/ap_bus.c

674

rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);

drivers/s390/crypto/ap_bus.c

678

rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);

drivers/s390/crypto/ap_bus.c

684

rc = add_uevent_var(env, "MODE=accel");

drivers/s390/crypto/ap_bus.c

686

rc = add_uevent_var(env, "MODE=cca");

drivers/s390/crypto/ap_bus.c

688

rc = add_uevent_var(env, "MODE=ep11");

drivers/s390/crypto/ap_bus.c

696

rc = add_uevent_var(env, "MODE=accel");

drivers/s390/crypto/ap_bus.c

698

rc = add_uevent_var(env, "MODE=cca");

drivers/s390/crypto/ap_bus.c

700

rc = add_uevent_var(env, "MODE=ep11");

drivers/s390/net/qeth_l2_main.c

1254

char *env[8];

drivers/s390/net/qeth_l2_main.c

1262

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1266

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1271

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1275

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1277

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1280

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1283

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1287

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1291

env[i] = str[i]; i++;

drivers/s390/net/qeth_l2_main.c

1294

env[i] = NULL;

drivers/s390/net/qeth_l2_main.c

1295

kobject_uevent_env(&card->gdev->dev.kobj, KOBJ_CHANGE, env);

drivers/s390/net/qeth_l2_main.c

1312

char *env[] = {

drivers/s390/net/qeth_l2_main.c

1331

KOBJ_CHANGE, env);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1288

struct kobj_uevent_env *env;

drivers/scsi/mpi3mr/mpi3mr_fw.c

1291

env = kzalloc_obj(*env);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1292

if (!env)

drivers/scsi/mpi3mr/mpi3mr_fw.c

1295

ret = add_uevent_var(env, "DRIVER=%s", mrioc->driver_name);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1299

ret = add_uevent_var(env, "IOC_ID=%u", mrioc->id);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1303

ret = add_uevent_var(env, "FAULT_CODE=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1308

ret = add_uevent_var(env, "FAULT_INFO0=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1313

ret = add_uevent_var(env, "FAULT_INFO1=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1318

ret = add_uevent_var(env, "FAULT_INFO2=0x%08x",

drivers/scsi/mpi3mr/mpi3mr_fw.c

1324

KOBJ_CHANGE, env->envp);

drivers/scsi/mpi3mr/mpi3mr_fw.c

1327

kfree(env);

drivers/scsi/pm8001/pm80xx_hwi.c

1562

struct kobj_uevent_env *env;

drivers/scsi/pm8001/pm80xx_hwi.c

1566

env = kzalloc_obj(struct kobj_uevent_env);

drivers/scsi/pm8001/pm80xx_hwi.c

1567

if (!env)

drivers/scsi/pm8001/pm80xx_hwi.c

1570

if (add_uevent_var(env, "DRIVER=%s", DRV_NAME))

drivers/scsi/pm8001/pm80xx_hwi.c

1573

if (add_uevent_var(env, "HBA_NUM=%u", pm8001_ha->id))

drivers/scsi/pm8001/pm80xx_hwi.c

1576

if (add_uevent_var(env, "EVENT_TYPE=FATAL_ERROR"))

drivers/scsi/pm8001/pm80xx_hwi.c

1581

if (add_uevent_var(env, "REPORTED_BY=DRIVER"))

drivers/scsi/pm8001/pm80xx_hwi.c

1585

if (add_uevent_var(env, "REPORTED_BY=FIRMWARE"))

drivers/scsi/pm8001/pm80xx_hwi.c

1589

if (add_uevent_var(env, "REPORTED_BY=OTHER"))

drivers/scsi/pm8001/pm80xx_hwi.c

1594

kobject_uevent_env(&pm8001_ha->shost->shost_dev.kobj, KOBJ_CHANGE, env->envp);

drivers/scsi/pm8001/pm80xx_hwi.c

1597

kfree(env);

drivers/scsi/scsi_sysfs.c

544

static int scsi_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/scsi/scsi_sysfs.c

553

add_uevent_var(env, "MODALIAS=" SCSI_DEVICE_MODALIAS_FMT, sdev->type);

drivers/slimbus/core.c

100

return add_uevent_var(env, "MODALIAS=slim:%s", dev_name(&sbdev->dev));

drivers/slimbus/core.c

96

static int slim_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soc/qcom/apr.c

390

static int apr_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soc/qcom/apr.c

395

ret = of_device_uevent_modalias(dev, env);

drivers/soc/qcom/apr.c

399

return add_uevent_var(env, "MODALIAS=apr:%s", adev->name);

drivers/soundwire/bus_type.c

62

int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/soundwire/bus_type.c

69

if (add_uevent_var(env, "MODALIAS=%s", modalias))

drivers/spi/spi.c

393

static int spi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/spi/spi.c

398

rc = acpi_device_uevent_modalias(dev, env);

drivers/spi/spi.c

402

return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);

drivers/spmi/spmi.c

370

static int spmi_drv_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/spmi/spmi.c

374

ret = of_device_uevent_modalias(dev, env);

drivers/ssb/main.c

342

static int ssb_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/ssb/main.c

351

return add_uevent_var(env,

drivers/staging/greybus/gbphy.c

74

static int gbphy_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/staging/greybus/gbphy.c

83

if (add_uevent_var(env, "BUS=%u", hd->bus_id))

drivers/staging/greybus/gbphy.c

85

if (add_uevent_var(env, "MODULE=%u", module->module_id))

drivers/staging/greybus/gbphy.c

87

if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))

drivers/staging/greybus/gbphy.c

89

if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",

drivers/staging/greybus/gbphy.c

92

if (add_uevent_var(env, "BUNDLE=%u", gbphy_dev->bundle->id))

drivers/staging/greybus/gbphy.c

94

if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))

drivers/staging/greybus/gbphy.c

96

if (add_uevent_var(env, "GBPHY=%u", gbphy_dev->id))

drivers/staging/greybus/gbphy.c

98

if (add_uevent_var(env, "PROTOCOL_ID=%02x", cport_desc->protocol_id))

drivers/staging/media/atomisp/pci/ia_css_control.h

37

const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/ia_css_device_access.c

16

ia_css_device_access_init(const struct ia_css_hw_access_env *env)

drivers/staging/media/atomisp/pci/ia_css_device_access.c

18

assert(env);

drivers/staging/media/atomisp/pci/ia_css_device_access.c

20

my_env = *env;

drivers/staging/media/atomisp/pci/ia_css_device_access.h

19

ia_css_device_access_init(const struct ia_css_hw_access_env *env);

drivers/staging/media/atomisp/pci/ia_css_firmware.h

42

ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1307

ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1312

if (!env)

drivers/staging/media/atomisp/pci/sh_css.c

1320

if (my_css.flush != env->cpu_mem_env.flush) {

drivers/staging/media/atomisp/pci/sh_css.c

1322

my_css.flush = env->cpu_mem_env.flush;

drivers/staging/media/atomisp/pci/sh_css.c

1334

ia_css_init(struct device *dev, const struct ia_css_env *env,

drivers/staging/media/atomisp/pci/sh_css.c

1342

if (!env)

drivers/staging/media/atomisp/pci/sh_css.c

1345

sh_css_printf = env->print_env.debug_print;

drivers/staging/media/atomisp/pci/sh_css.c

1349

flush_func = env->cpu_mem_env.flush;

drivers/staging/media/atomisp/pci/sh_css.c

1355

ia_css_device_access_init(&env->hw_access_env);

drivers/staging/media/atomisp/pci/sh_css.c

1365

my_css_save.driver_env = *env;

drivers/staging/media/ipu3/ipu3-css.c

1699

struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];

drivers/staging/media/ipu3/ipu3-css.c

1784

env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;

drivers/staging/media/ipu3/ipu3-css.c

1786

env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;

drivers/tee/tee_core.c

1440

struct kobj_uevent_env *env)

drivers/tee/tee_core.c

1444

return add_uevent_var(env, "MODALIAS=tee:%pUb", dev_id);

drivers/thunderbolt/switch.c

2314

static int tb_switch_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/thunderbolt/switch.c

2320

if (add_uevent_var(env, "USB4_VERSION=%u.0",

drivers/thunderbolt/switch.c

2343

if (add_uevent_var(env, "USB4_TYPE=%s", type))

drivers/thunderbolt/xdomain.c

989

static int tb_service_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/thunderbolt/xdomain.c

995

return add_uevent_var(env, "MODALIAS=%s", modalias);

drivers/tty/serdev/core.c

47

static int serdev_device_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/tty/serdev/core.c

53

rc = acpi_device_uevent_modalias(dev, env);

drivers/tty/serdev/core.c

57

return of_device_uevent_modalias(dev, env);

drivers/usb/common/ulpi.c

58

static int ulpi_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/common/ulpi.c

63

ret = of_device_uevent_modalias(dev, env);

drivers/usb/common/ulpi.c

67

if (add_uevent_var(env, "MODALIAS=ulpi:v%04xp%04x",

drivers/usb/core/driver.c

915

static int usb_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/driver.c

940

if (add_uevent_var(env, "PRODUCT=%x/%x/%x",

drivers/usb/core/driver.c

947

if (add_uevent_var(env, "TYPE=%d/%d/%d",

drivers/usb/core/hcd.c

2489

static char *env[] = {

drivers/usb/core/hcd.c

2495

kobject_uevent_env(&hcd->self.root_hub->dev.kobj, KOBJ_OFFLINE, env);

drivers/usb/core/message.c

1879

static int usb_if_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/message.c

1889

if (add_uevent_var(env, "INTERFACE=%d/%d/%d",

drivers/usb/core/message.c

1895

if (add_uevent_var(env,

drivers/usb/core/usb.c

504

static int usb_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/core/usb.c

510

if (add_uevent_var(env, "BUSNUM=%03d", usb_dev->bus->busnum))

drivers/usb/core/usb.c

513

if (add_uevent_var(env, "DEVNUM=%03d", usb_dev->devnum))

drivers/usb/gadget/udc/core.c

1889

static int usb_udc_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/gadget/udc/core.c

1894

ret = add_uevent_var(env, "USB_UDC_NAME=%s", udc->gadget->name);

drivers/usb/gadget/udc/core.c

1902

ret = add_uevent_var(env, "USB_UDC_DRIVER=%s",

drivers/usb/phy/phy.c

148

static int usb_phy_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/phy/phy.c

168

if (add_uevent_var(env, uchger_state))

drivers/usb/phy/phy.c

171

if (add_uevent_var(env, uchger_type))

drivers/usb/roles/class.c

321

static int usb_role_switch_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/roles/class.c

325

ret = add_uevent_var(env, "USB_ROLE_SWITCH=%s", dev_name(dev));

drivers/usb/typec/bus.c

481

static int typec_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/typec/bus.c

488

if (add_uevent_var(env, "SVID=%04X", altmode->svid))

drivers/usb/typec/bus.c

491

if (add_uevent_var(env, "MODE=%u", altmode->mode))

drivers/usb/typec/bus.c

494

return add_uevent_var(env, "MODALIAS=typec:id%04X", altmode->svid);

drivers/usb/typec/class.c

2098

static int typec_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/usb/typec/class.c

2102

ret = add_uevent_var(env, "TYPEC_PORT=%s", dev_name(dev));

drivers/virtio/virtio.c

102

return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",

drivers/virtio/virtio.c

98

static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env)

drivers/w1/w1.c

168

static int w1_uevent(const struct device *dev, struct kobj_uevent_env *env);

drivers/w1/w1.c

570

static int w1_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/w1/w1.c

596

err = add_uevent_var(env, "W1_FID=%02X", sl->reg_num.family);

drivers/w1/w1.c

600

err = add_uevent_var(env, "W1_SLAVE_ID=%024LX",

drivers/xen/pvcalls-back.c

1198

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_backend.c

110

if (add_uevent_var(env, "MODALIAS=xen-backend:%s", xdev->devicetype))

drivers/xen/xenbus/xenbus_probe_backend.c

114

if (add_uevent_var(env, "XENBUS_TYPE=%s", xdev->devicetype))

drivers/xen/xenbus/xenbus_probe_backend.c

117

if (add_uevent_var(env, "XENBUS_PATH=%s", xdev->nodename))

drivers/xen/xenbus/xenbus_probe_backend.c

120

if (add_uevent_var(env, "XENBUS_BASE_PATH=%s", bus->root))

drivers/xen/xenbus/xenbus_probe_backend.c

126

return drv->uevent(xdev, env);

drivers/xen/xenbus/xenbus_probe_backend.c

96

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_frontend.c

77

struct kobj_uevent_env *env)

drivers/xen/xenbus/xenbus_probe_frontend.c

81

if (add_uevent_var(env, "MODALIAS=xen:%s", dev->devicetype))

drivers/zorro/zorro-driver.c

133

static int zorro_uevent(const struct device *dev, struct kobj_uevent_env *env)

drivers/zorro/zorro-driver.c

144

if (add_uevent_var(env, "ZORRO_ID=%08X", z->id) ||

drivers/zorro/zorro-driver.c

145

add_uevent_var(env, "ZORRO_SLOT_NAME=%s", dev_name(dev)) ||

drivers/zorro/zorro-driver.c

146

add_uevent_var(env, "ZORRO_SLOT_ADDR=%04X", z->slotaddr) ||

drivers/zorro/zorro-driver.c

147

add_uevent_var(env, "MODALIAS=" ZORRO_DEVICE_MODALIAS_FMT, z->id))

fs/dlm/lockspace.c

214

static int dlm_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/dlm/lockspace.c

218

add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name);

fs/fuse/virtio_fs.c

1762

static int virtio_fs_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/fuse/virtio_fs.c

1766

add_uevent_var(env, "TAG=%s", fs->tag);

fs/gfs2/sys.c

767

static int gfs2_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

fs/gfs2/sys.c

772

add_uevent_var(env, "LOCKTABLE=%s", sdp->sd_table_name);

fs/gfs2/sys.c

773

add_uevent_var(env, "LOCKPROTO=%s", sdp->sd_proto_name);

fs/gfs2/sys.c

775

add_uevent_var(env, "JOURNALID=%d", sdp->sd_lockstruct.ls_jid);

fs/gfs2/sys.c

777

add_uevent_var(env, "UUID=%pUB", &s->s_uuid);

fs/hfsplus/unicode_test.c

23

struct test_mock_string_env *env;

fs/hfsplus/unicode_test.c

25

env = kzalloc_obj(struct test_mock_string_env);

fs/hfsplus/unicode_test.c

26

if (!env)

fs/hfsplus/unicode_test.c

29

env->buf = kzalloc(buf_size, GFP_KERNEL);

fs/hfsplus/unicode_test.c

30

if (!env->buf) {

fs/hfsplus/unicode_test.c

31

kfree(env);

fs/hfsplus/unicode_test.c

35

env->buf_size = buf_size;

fs/hfsplus/unicode_test.c

37

return env;

fs/hfsplus/unicode_test.c

40

static void free_mock_str_env(struct test_mock_string_env *env)

fs/hfsplus/unicode_test.c

42

if (env->buf)

fs/hfsplus/unicode_test.c

43

kfree(env->buf);

fs/hfsplus/unicode_test.c

44

kfree(env);

include/linux/acpi.h

1003

struct kobj_uevent_env *env)

include/linux/bpf.h

1127

int (*insn_hook)(struct bpf_verifier_env *env,

include/linux/bpf.h

1129

int (*finalize)(struct bpf_verifier_env *env);

include/linux/bpf.h

1131

int (*replace_insn)(struct bpf_verifier_env *env, u32 off,

include/linux/bpf.h

1133

int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);

include/linux/bpf.h

173

int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,

include/linux/bpf.h

2895

int map_set_for_each_callback_args(struct bpf_verifier_env *env,

include/linux/bpf.h

3024

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);

include/linux/bpf_verifier.h

1017

static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)

include/linux/bpf_verifier.h

1019

env->scratched_regs |= 1U << regno;

include/linux/bpf_verifier.h

1022

static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)

include/linux/bpf_verifier.h

1024

env->scratched_stack_slots |= 1ULL << spi;

include/linux/bpf_verifier.h

1027

static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)

include/linux/bpf_verifier.h

1029

return (env->scratched_regs >> regno) & 1;

include/linux/bpf_verifier.h

1032

static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)

include/linux/bpf_verifier.h

1034

return (env->scratched_stack_slots >> regno) & 1;

include/linux/bpf_verifier.h

1037

static inline bool verifier_state_scratched(const struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1039

return env->scratched_regs || env->scratched_stack_slots;

include/linux/bpf_verifier.h

1042

static inline void mark_verifier_state_clean(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1044

env->scratched_regs = 0U;

include/linux/bpf_verifier.h

1045

env->scratched_stack_slots = 0ULL;

include/linux/bpf_verifier.h

1049

static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

1051

env->scratched_regs = ~0U;

include/linux/bpf_verifier.h

1052

env->scratched_stack_slots = ~0ULL;

include/linux/bpf_verifier.h

1071

const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type);

include/linux/bpf_verifier.h

1076

void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

include/linux/bpf_verifier.h

1078

void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

include/linux/bpf_verifier.h

1081

struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);

include/linux/bpf_verifier.h

1083

struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);

include/linux/bpf_verifier.h

1085

bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);

include/linux/bpf_verifier.h

1087

int bpf_stack_liveness_init(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1088

void bpf_stack_liveness_free(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1089

int bpf_update_live_stack(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1090

int bpf_mark_stack_read(struct bpf_verifier_env *env, u32 frameno, u32 insn_idx, u64 mask);

include/linux/bpf_verifier.h

1091

void bpf_mark_stack_write(struct bpf_verifier_env *env, u32 frameno, u64 mask);

include/linux/bpf_verifier.h

1092

int bpf_reset_stack_write_marks(struct bpf_verifier_env *env, u32 insn_idx);

include/linux/bpf_verifier.h

1093

int bpf_commit_stack_write_marks(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

1094

int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);

include/linux/bpf_verifier.h

1095

bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);

include/linux/bpf_verifier.h

1096

void bpf_reset_live_stack_callchain(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

686

struct bpf_verifier_env *env;

include/linux/bpf_verifier.h

856

static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

858

return &env->prog->aux->func_info_aux[subprog];

include/linux/bpf_verifier.h

861

static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog)

include/linux/bpf_verifier.h

863

return &env->subprog_info[subprog];

include/linux/bpf_verifier.h

868

__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

877

__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

881

#define verifier_bug_if(cond, env, fmt, args...) \

include/linux/bpf_verifier.h

885

verifier_bug(env, fmt " (" #cond ")", ##args); \

include/linux/bpf_verifier.h

888

#define verifier_bug(env, fmt, args...) \

include/linux/bpf_verifier.h

891

bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \

include/linux/bpf_verifier.h

894

static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

896

struct bpf_verifier_state *cur = env->cur_state;

include/linux/bpf_verifier.h

901

static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env)

include/linux/bpf_verifier.h

903

return cur_func(env)->regs;

include/linux/bpf_verifier.h

907

int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,

include/linux/bpf_verifier.h

909

int bpf_prog_offload_finalize(struct bpf_verifier_env *env);

include/linux/bpf_verifier.h

911

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

include/linux/bpf_verifier.h

914

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);

include/linux/bpf_verifier.h

942

int mark_chain_precision(struct bpf_verifier_env *env, int regno);

include/linux/device.h

91

int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/device/bus.h

91

int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/device/class.h

56

int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env);

include/linux/ieee80211-he.h

643

const struct ieee80211_tx_pwr_env *env = (const void *)data;

include/linux/ieee80211-he.h

645

u8 needed = sizeof(*env);

include/linux/ieee80211-he.h

651

count = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_COUNT);

include/linux/ieee80211-he.h

652

interpret = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_INTERPRET);

include/linux/ieee80211-he.h

653

category = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_CATEGORY);

include/linux/ieee80211-he.h

690

u8 K = u8_get_bits(env->variable[N],

include/linux/kobject.h

136

int (* const uevent)(const struct kobject *kobj, struct kobj_uevent_env *env);

include/linux/kobject.h

220

int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...);

include/linux/of_device.h

28

extern void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env);

include/linux/of_device.h

29

extern int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env);

include/linux/of_device.h

52

struct kobj_uevent_env *env) { }

include/linux/of_device.h

61

struct kobj_uevent_env *env)

include/linux/soundwire/sdw_type.h

24

int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env);

kernel/bpf/btf.c

1545

__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

1548

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1559

__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,

kernel/bpf/btf.c

1564

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1565

struct btf *btf = env->btf;

kernel/bpf/btf.c

1580

if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))

kernel/bpf/btf.c

1585

env->log_type_id,

kernel/bpf/btf.c

1591

btf_type_ops(t)->log_details(env, t);

kernel/bpf/btf.c

1603

#define btf_verifier_log_type(env, t, ...) \

kernel/bpf/btf.c

1604

__btf_verifier_log_type((env), (t), true, __VA_ARGS__)

kernel/bpf/btf.c

1605

#define btf_verifier_log_basic(env, t, ...) \

kernel/bpf/btf.c

1606

__btf_verifier_log_type((env), (t), false, __VA_ARGS__)

kernel/bpf/btf.c

1609

static void btf_verifier_log_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

1614

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1615

struct btf *btf = env->btf;

kernel/bpf/btf.c

1626

if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH))

kernel/bpf/btf.c

1636

if (env->phase != CHECK_META)

kernel/bpf/btf.c

1637

btf_verifier_log_type(env, struct_type, NULL);

kernel/bpf/btf.c

1662

static void btf_verifier_log_vsi(struct btf_verifier_env *env,

kernel/bpf/btf.c

1667

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1674

if (env->phase != CHECK_META)

kernel/bpf/btf.c

1675

btf_verifier_log_type(env, datasec_type, NULL);

kernel/bpf/btf.c

1689

static void btf_verifier_log_hdr(struct btf_verifier_env *env,

kernel/bpf/btf.c

1692

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

1693

const struct btf *btf = env->btf;

kernel/bpf/btf.c

1713

static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)

kernel/bpf/btf.c

1715

struct btf *btf = env->btf;

kernel/bpf/btf.c

1724

btf_verifier_log(env, "Exceeded max num of types");

kernel/bpf/btf.c

1920

static int env_resolve_init(struct btf_verifier_env *env)

kernel/bpf/btf.c

1922

struct btf *btf = env->btf;

kernel/bpf/btf.c

1945

env->visit_states = visit_states;

kernel/bpf/btf.c

1956

static void btf_verifier_env_free(struct btf_verifier_env *env)

kernel/bpf/btf.c

1958

kvfree(env->visit_states);

kernel/bpf/btf.c

1959

kfree(env);

kernel/bpf/btf.c

1962

static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,

kernel/bpf/btf.c

1965

switch (env->resolve_mode) {

kernel/bpf/btf.c

1987

static bool env_type_is_resolved(const struct btf_verifier_env *env,

kernel/bpf/btf.c

1991

if (type_id < env->btf->start_id)

kernel/bpf/btf.c

1994

return env->visit_states[type_id - env->btf->start_id] == RESOLVED;

kernel/bpf/btf.c

1997

static int env_stack_push(struct btf_verifier_env *env,

kernel/bpf/btf.c

2000

const struct btf *btf = env->btf;

kernel/bpf/btf.c

2003

if (env->top_stack == MAX_RESOLVE_DEPTH)

kernel/bpf/btf.c

2007

|| env->visit_states[type_id - btf->start_id] != NOT_VISITED)

kernel/bpf/btf.c

2010

env->visit_states[type_id - btf->start_id] = VISITED;

kernel/bpf/btf.c

2012

v = &env->stack[env->top_stack++];

kernel/bpf/btf.c

2017

if (env->resolve_mode == RESOLVE_TBD) {

kernel/bpf/btf.c

2019

env->resolve_mode = RESOLVE_PTR;

kernel/bpf/btf.c

2021

env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;

kernel/bpf/btf.c

2027

static void env_stack_set_next_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2030

env->stack[env->top_stack - 1].next_member = next_member;

kernel/bpf/btf.c

2033

static void env_stack_pop_resolved(struct btf_verifier_env *env,

kernel/bpf/btf.c

2037

u32 type_id = env->stack[--(env->top_stack)].type_id;

kernel/bpf/btf.c

2038

struct btf *btf = env->btf;

kernel/bpf/btf.c

2043

env->visit_states[type_id] = RESOLVED;

kernel/bpf/btf.c

2046

static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)

kernel/bpf/btf.c

2048

return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;

kernel/bpf/btf.c

2215

static int btf_df_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2220

btf_verifier_log_basic(env, struct_type,

kernel/bpf/btf.c

2225

static int btf_df_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2230

btf_verifier_log_basic(env, struct_type,

kernel/bpf/btf.c

2238

static int btf_generic_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2244

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2252

return btf_type_ops(member_type)->check_member(env, struct_type,

kernel/bpf/btf.c

2257

static int btf_df_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2260

btf_verifier_log_basic(env, v->t, "Unsupported resolve");

kernel/bpf/btf.c

2271

static int btf_int_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2283

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2294

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2301

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2309

static int btf_int_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2321

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2335

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2342

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2350

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2357

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2365

static s32 btf_int_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2373

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

2380

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2385

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

2391

btf_verifier_log_basic(env, t, "Invalid int_data:%x",

kernel/bpf/btf.c

2399

btf_verifier_log_type(env, t, "nr_bits exceeds %zu",

kernel/bpf/btf.c

2405

btf_verifier_log_type(env, t, "nr_bits exceeds type_size");

kernel/bpf/btf.c

2420

btf_verifier_log_type(env, t, "Unsupported encoding");

kernel/bpf/btf.c

2424

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2429

static void btf_int_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2434

btf_verifier_log(env,

kernel/bpf/btf.c

2630

static int btf_modifier_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2638

struct btf *btf = env->btf;

kernel/bpf/btf.c

2642

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2650

return btf_type_ops(resolved_type)->check_member(env, struct_type,

kernel/bpf/btf.c

2655

static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2663

struct btf *btf = env->btf;

kernel/bpf/btf.c

2667

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2675

return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,

kernel/bpf/btf.c

2680

static int btf_ptr_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

2692

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2698

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

2706

static int btf_ref_type_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2713

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2718

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

2723

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

2732

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

2733

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2737

value = btf_name_by_offset(env->btf, t->name_off);

kernel/bpf/btf.c

2739

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2744

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2749

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2754

static int btf_modifier_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2760

struct btf *btf = env->btf;

kernel/bpf/btf.c

2764

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2768

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2769

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2770

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2779

if (env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2786

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2791

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2796

static int btf_var_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2802

struct btf *btf = env->btf;

kernel/bpf/btf.c

2806

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2810

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2811

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2812

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2822

!env_type_is_resolve_sink(env, resolved_type) &&

kernel/bpf/btf.c

2823

!env_type_is_resolved(env, resolved_type_id))

kernel/bpf/btf.c

2824

return env_stack_push(env, resolved_type,

kernel/bpf/btf.c

2833

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2837

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2842

static int btf_ptr_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

2848

struct btf *btf = env->btf;

kernel/bpf/btf.c

2852

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2856

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

2857

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2858

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

2876

!env_type_is_resolve_sink(env, resolved_type) &&

kernel/bpf/btf.c

2877

!env_type_is_resolved(env, resolved_type_id))

kernel/bpf/btf.c

2878

return env_stack_push(env, resolved_type,

kernel/bpf/btf.c

2883

if (env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

2889

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

2894

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

2939

static void btf_ref_type_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2942

btf_verifier_log(env, "type_id=%u", t->type);

kernel/bpf/btf.c

2963

static s32 btf_fwd_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

2968

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

2973

btf_verifier_log_type(env, t, "type != 0");

kernel/bpf/btf.c

2979

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

2980

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

2984

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

2989

static void btf_fwd_type_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

2992

btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");

kernel/bpf/btf.c

3004

static int btf_array_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

3012

struct btf *btf = env->btf;

kernel/bpf/btf.c

3015

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3025

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3033

static s32 btf_array_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

3041

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

3049

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

3054

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

3059

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

3064

btf_verifier_log_type(env, t, "size != 0");

kernel/bpf/btf.c

3072

btf_verifier_log_type(env, t, "Invalid elem");

kernel/bpf/btf.c

3077

btf_verifier_log_type(env, t, "Invalid index");

kernel/bpf/btf.c

3081

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

3086

static int btf_array_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

3092

struct btf *btf = env->btf;

kernel/bpf/btf.c

3100

btf_verifier_log_type(env, v->t, "Invalid index");

kernel/bpf/btf.c

3104

if (!env_type_is_resolve_sink(env, index_type) &&

kernel/bpf/btf.c

3105

!env_type_is_resolved(env, index_type_id))

kernel/bpf/btf.c

3106

return env_stack_push(env, index_type, index_type_id);

kernel/bpf/btf.c

3111

btf_verifier_log_type(env, v->t, "Invalid index");

kernel/bpf/btf.c

3120

btf_verifier_log_type(env, v->t,

kernel/bpf/btf.c

3125

if (!env_type_is_resolve_sink(env, elem_type) &&

kernel/bpf/btf.c

3126

!env_type_is_resolved(env, elem_type_id))

kernel/bpf/btf.c

3127

return env_stack_push(env, elem_type, elem_type_id);

kernel/bpf/btf.c

3131

btf_verifier_log_type(env, v->t, "Invalid elem");

kernel/bpf/btf.c

3136

btf_verifier_log_type(env, v->t, "Invalid array of int");

kernel/bpf/btf.c

3141

btf_verifier_log_type(env, v->t,

kernel/bpf/btf.c

3146

env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);

kernel/bpf/btf.c

3151

static void btf_array_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

3156

btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",

kernel/bpf/btf.c

3255

static int btf_struct_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

3264

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3272

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

3280

static s32 btf_struct_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

3287

struct btf *btf = env->btf;

kernel/bpf/btf.c

3294

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

3302

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

3303

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

3307

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

3312

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3321

btf_verifier_log_member(env, t, member, "Invalid name");

kernel/bpf/btf.c

3326

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3333

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3343

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3349

btf_verifier_log_member(env, t, member,

kernel/bpf/btf.c

3354

btf_verifier_log_member(env, t, member, NULL);

kernel/bpf/btf.c

3361

static int btf_struct_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

3379

if (WARN_ON_ONCE(!env_type_is_resolved(env,

kernel/bpf/btf.c

3383

last_member_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

3386

err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,

kernel/bpf/btf.c

3390

err = btf_type_ops(last_member_type)->check_member(env, v->t,

kernel/bpf/btf.c

3399

const struct btf_type *member_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

3404

btf_verifier_log_member(env, v->t, member,

kernel/bpf/btf.c

3409

if (!env_type_is_resolve_sink(env, member_type) &&

kernel/bpf/btf.c

3410

!env_type_is_resolved(env, member_type_id)) {

kernel/bpf/btf.c

3411

env_stack_set_next_member(env, i + 1);

kernel/bpf/btf.c

3412

return env_stack_push(env, member_type, member_type_id);

kernel/bpf/btf.c

3416

err = btf_type_ops(member_type)->check_kflag_member(env, v->t,

kernel/bpf/btf.c

3420

err = btf_type_ops(member_type)->check_member(env, v->t,

kernel/bpf/btf.c

3427

env_stack_pop_resolved(env, 0, 0);

kernel/bpf/btf.c

3432

static void btf_struct_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

3435

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

4377

static int btf_enum_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

4386

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4394

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4402

static int btf_enum_check_kflag_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

4414

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4421

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4429

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

4437

static s32 btf_enum_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4442

struct btf *btf = env->btf;

kernel/bpf/btf.c

4451

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4458

btf_verifier_log_type(env, t, "Unexpected size");

kernel/bpf/btf.c

446

s32 (*check_meta)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4464

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4465

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4469

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4473

btf_verifier_log(env, "\tInvalid name_offset:%u",

kernel/bpf/btf.c

4481

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4485

if (env->log.level == BPF_LOG_KERNEL)

kernel/bpf/btf.c

4488

btf_verifier_log(env, fmt_str,

kernel/bpf/btf.c

449

int (*resolve)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4496

static void btf_enum_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

4499

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

451

int (*check_member)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4545

static s32 btf_enum64_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

455

int (*check_kflag_member)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4550

struct btf *btf = env->btf;

kernel/bpf/btf.c

4559

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4566

btf_verifier_log_type(env, t, "Unexpected size");

kernel/bpf/btf.c

4572

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4573

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4577

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4581

btf_verifier_log(env, "\tInvalid name_offset:%u",

kernel/bpf/btf.c

4589

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

459

void (*log_details)(struct btf_verifier_env *env,

kernel/bpf/btf.c

4593

if (env->log.level == BPF_LOG_KERNEL)

kernel/bpf/btf.c

4597

btf_verifier_log(env, fmt_str,

kernel/bpf/btf.c

4648

static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4655

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4662

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4667

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4671

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4676

static void btf_func_proto_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

4682

btf_verifier_log(env, "return=%u args=(", t->type);

kernel/bpf/btf.c

4684

btf_verifier_log(env, "void");

kernel/bpf/btf.c

469

static int btf_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4690

btf_verifier_log(env, "vararg");

kernel/bpf/btf.c

4694

btf_verifier_log(env, "%u %s", args[0].type,

kernel/bpf/btf.c

4695

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4698

btf_verifier_log(env, ", %u %s", args[i].type,

kernel/bpf/btf.c

4699

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4706

btf_verifier_log(env, ", %u %s", last_arg->type,

kernel/bpf/btf.c

4707

__btf_name_by_offset(env->btf,

kernel/bpf/btf.c

4710

btf_verifier_log(env, ", vararg");

kernel/bpf/btf.c

4714

btf_verifier_log(env, ")");

kernel/bpf/btf.c

472

static int btf_func_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

4735

static s32 btf_func_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4740

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4741

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4746

btf_verifier_log_type(env, t, "Invalid func linkage");

kernel/bpf/btf.c

4751

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4755

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4760

static int btf_func_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4767

err = btf_func_check(env, t);

kernel/bpf/btf.c

4771

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

4784

static s32 btf_var_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4792

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4799

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

4804

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4809

!btf_name_valid_identifier(env->btf, t->name_off)) {

kernel/bpf/btf.c

4810

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4816

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

4823

btf_verifier_log_type(env, t, "Linkage not supported");

kernel/bpf/btf.c

4827

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4832

static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t)

kernel/bpf/btf.c

4836

btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);

kernel/bpf/btf.c

4848

static s32 btf_datasec_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

4858

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

4865

btf_verifier_log_type(env, t, "size == 0");

kernel/bpf/btf.c

4870

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

4875

!btf_name_valid_section(env->btf, t->name_off)) {

kernel/bpf/btf.c

4876

btf_verifier_log_type(env, t, "Invalid name");

kernel/bpf/btf.c

4880

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

4885

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4891

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4897

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4904

btf_verifier_log_vsi(env, t, vsi,

kernel/bpf/btf.c

4909

btf_verifier_log_vsi(env, t, vsi, NULL);

kernel/bpf/btf.c

4914

btf_verifier_log_type(env, t, "Invalid btf_info size");

kernel/bpf/btf.c

4921

static int btf_datasec_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

4925

struct btf *btf = env->btf;

kernel/bpf/btf.c

4928

env->resolve_mode = RESOLVE_TBD;

kernel/bpf/btf.c

4931

const struct btf_type *var_type = btf_type_by_id(env->btf,

kernel/bpf/btf.c

4934

btf_verifier_log_vsi(env, v->t, vsi,

kernel/bpf/btf.c

4939

if (!env_type_is_resolve_sink(env, var_type) &&

kernel/bpf/btf.c

4940

!env_type_is_resolved(env, var_type_id)) {

kernel/bpf/btf.c

4941

env_stack_set_next_member(env, i + 1);

kernel/bpf/btf.c

4942

return env_stack_push(env, var_type, var_type_id);

kernel/bpf/btf.c

4947

btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");

kernel/bpf/btf.c

4952

btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");

kernel/bpf/btf.c

4957

env_stack_pop_resolved(env, 0, 0);

kernel/bpf/btf.c

4961

static void btf_datasec_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

4964

btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));

kernel/bpf/btf.c

5000

static s32 btf_float_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5005

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

5010

btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");

kernel/bpf/btf.c

5016

btf_verifier_log_type(env, t, "Invalid type_size");

kernel/bpf/btf.c

5020

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

5025

static int btf_float_check_member(struct btf_verifier_env *env,

kernel/bpf/btf.c

5044

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

5052

btf_verifier_log_member(env, struct_type, member,

kernel/bpf/btf.c

5060

static void btf_float_log(struct btf_verifier_env *env,

kernel/bpf/btf.c

5063

btf_verifier_log(env, "size=%u", t->size);

kernel/bpf/btf.c

5075

static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5085

btf_verifier_log_basic(env, t,

kernel/bpf/btf.c

5091

value = btf_name_by_offset(env->btf, t->name_off);

kernel/bpf/btf.c

5093

btf_verifier_log_type(env, t, "Invalid value");

kernel/bpf/btf.c

5098

btf_verifier_log_type(env, t, "vlen != 0");

kernel/bpf/btf.c

5104

btf_verifier_log_type(env, t, "Invalid component_idx");

kernel/bpf/btf.c

5108

btf_verifier_log_type(env, t, NULL);

kernel/bpf/btf.c

5113

static int btf_decl_tag_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

5119

struct btf *btf = env->btf;

kernel/bpf/btf.c

5125

btf_verifier_log_type(env, v->t, "Invalid type_id");

kernel/bpf/btf.c

5129

if (!env_type_is_resolve_sink(env, next_type) &&

kernel/bpf/btf.c

5130

!env_type_is_resolved(env, next_type_id))

kernel/bpf/btf.c

5131

return env_stack_push(env, next_type, next_type_id);

kernel/bpf/btf.c

5136

btf_verifier_log_type(env, v->t, "Invalid component_idx");

kernel/bpf/btf.c

5149

btf_verifier_log_type(env, v->t, "Invalid component_idx");

kernel/bpf/btf.c

5154

env_stack_pop_resolved(env, next_type_id, 0);

kernel/bpf/btf.c

5159

static void btf_decl_tag_log(struct btf_verifier_env *env, const struct btf_type *t)

kernel/bpf/btf.c

5161

btf_verifier_log(env, "type=%u component_idx=%d", t->type,

kernel/bpf/btf.c

5174

static int btf_func_proto_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

5183

btf = env->btf;

kernel/bpf/btf.c

5193

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5198

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5203

!env_type_is_resolved(env, ret_type_id)) {

kernel/bpf/btf.c

5204

err = btf_resolve(env, ret_type, ret_type_id);

kernel/bpf/btf.c

5211

btf_verifier_log_type(env, t, "Invalid return type");

kernel/bpf/btf.c

5222

btf_verifier_log_type(env, t, "Invalid arg#%u",

kernel/bpf/btf.c

5236

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5241

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5248

btf_verifier_log_type(env, t,

kernel/bpf/btf.c

5254

!env_type_is_resolved(env, arg_type_id)) {

kernel/bpf/btf.c

5255

err = btf_resolve(env, arg_type, arg_type_id);

kernel/bpf/btf.c

5261

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5269

static int btf_func_check(struct btf_verifier_env *env,

kernel/bpf/btf.c

5277

btf = env->btf;

kernel/bpf/btf.c

5281

btf_verifier_log_type(env, t, "Invalid type_id");

kernel/bpf/btf.c

5289

btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);

kernel/bpf/btf.c

5319

static s32 btf_check_meta(struct btf_verifier_env *env,

kernel/bpf/btf.c

5327

btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",

kernel/bpf/btf.c

5328

env->log_type_id, meta_left, sizeof(*t));

kernel/bpf/btf.c

5334

btf_verifier_log(env, "[%u] Invalid btf_info:%x",

kernel/bpf/btf.c

5335

env->log_type_id, t->info);

kernel/bpf/btf.c

5341

btf_verifier_log(env, "[%u] Invalid kind:%u",

kernel/bpf/btf.c

5342

env->log_type_id, BTF_INFO_KIND(t->info));

kernel/bpf/btf.c

5346

if (!btf_name_offset_valid(env->btf, t->name_off)) {

kernel/bpf/btf.c

5347

btf_verifier_log(env, "[%u] Invalid name_offset:%u",

kernel/bpf/btf.c

5348

env->log_type_id, t->name_off);

kernel/bpf/btf.c

5352

var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);

kernel/bpf/btf.c

5361

static int btf_check_all_metas(struct btf_verifier_env *env)

kernel/bpf/btf.c

5363

struct btf *btf = env->btf;

kernel/bpf/btf.c

5371

env->log_type_id = btf->base_btf ? btf->start_id : 1;

kernel/bpf/btf.c

5376

meta_size = btf_check_meta(env, t, end - cur);

kernel/bpf/btf.c

5380

btf_add_type(env, t);

kernel/bpf/btf.c

5382

env->log_type_id++;

kernel/bpf/btf.c

5388

static bool btf_resolve_valid(struct btf_verifier_env *env,

kernel/bpf/btf.c

5392

struct btf *btf = env->btf;

kernel/bpf/btf.c

5394

if (!env_type_is_resolved(env, type_id))

kernel/bpf/btf.c

5429

static int btf_resolve(struct btf_verifier_env *env,

kernel/bpf/btf.c

5432

u32 save_log_type_id = env->log_type_id;

kernel/bpf/btf.c

5436

env->resolve_mode = RESOLVE_TBD;

kernel/bpf/btf.c

5437

env_stack_push(env, t, type_id);

kernel/bpf/btf.c

5438

while (!err && (v = env_stack_peak(env))) {

kernel/bpf/btf.c

5439

env->log_type_id = v->type_id;

kernel/bpf/btf.c

5440

err = btf_type_ops(v->t)->resolve(env, v);

kernel/bpf/btf.c

5443

env->log_type_id = type_id;

kernel/bpf/btf.c

5445

btf_verifier_log_type(env, t,

kernel/bpf/btf.c

5449

btf_verifier_log_type(env, t, "Loop detected");

kernel/bpf/btf.c

5453

if (!err && !btf_resolve_valid(env, t, type_id)) {

kernel/bpf/btf.c

5454

btf_verifier_log_type(env, t, "Invalid resolve state");

kernel/bpf/btf.c

5458

env->log_type_id = save_log_type_id;

kernel/bpf/btf.c

5462

static int btf_check_all_types(struct btf_verifier_env *env)

kernel/bpf/btf.c

5464

struct btf *btf = env->btf;

kernel/bpf/btf.c

5469

err = env_resolve_init(env);

kernel/bpf/btf.c

5473

env->phase++;

kernel/bpf/btf.c

5478

env->log_type_id = type_id;

kernel/bpf/btf.c

5480

!env_type_is_resolved(env, type_id)) {

kernel/bpf/btf.c

5481

err = btf_resolve(env, t, type_id);

kernel/bpf/btf.c

5487

err = btf_func_proto_check(env, t);

kernel/bpf/btf.c

5496

static int btf_parse_type_sec(struct btf_verifier_env *env)

kernel/bpf/btf.c

5498

const struct btf_header *hdr = &env->btf->hdr;

kernel/bpf/btf.c

5503

btf_verifier_log(env, "Unaligned type_off");

kernel/bpf/btf.c

5507

if (!env->btf->base_btf && !hdr->type_len) {

kernel/bpf/btf.c

5508

btf_verifier_log(env, "No type found");

kernel/bpf/btf.c

5512

err = btf_check_all_metas(env);

kernel/bpf/btf.c

5516

return btf_check_all_types(env);

kernel/bpf/btf.c

5519

static int btf_parse_str_sec(struct btf_verifier_env *env)

kernel/bpf/btf.c

5522

struct btf *btf = env->btf;

kernel/bpf/btf.c

5530

btf_verifier_log(env, "String section is not at the end");

kernel/bpf/btf.c

5539

btf_verifier_log(env, "Invalid string section");

kernel/bpf/btf.c

5543

btf_verifier_log(env, "Invalid string section");

kernel/bpf/btf.c

5563

static int btf_check_sec_info(struct btf_verifier_env *env,

kernel/bpf/btf.c

5571

btf = env->btf;

kernel/bpf/btf.c

5587

btf_verifier_log(env, "Invalid section offset");

kernel/bpf/btf.c

5592

btf_verifier_log(env, "Unsupported section found");

kernel/bpf/btf.c

5596

btf_verifier_log(env, "Section overlap found");

kernel/bpf/btf.c

5600

btf_verifier_log(env,

kernel/bpf/btf.c

5609

btf_verifier_log(env, "Unsupported section found");

kernel/bpf/btf.c

5616

static int btf_parse_hdr(struct btf_verifier_env *env)

kernel/bpf/btf.c

5622

btf = env->btf;

kernel/bpf/btf.c

5626

btf_verifier_log(env, "hdr_len not found");

kernel/bpf/btf.c

5633

btf_verifier_log(env, "btf_header not found");

kernel/bpf/btf.c

5644

btf_verifier_log(env, "Unsupported btf_header");

kernel/bpf/btf.c

5655

btf_verifier_log_hdr(env, btf_data_size);

kernel/bpf/btf.c

5658

btf_verifier_log(env, "Invalid magic");

kernel/bpf/btf.c

5663

btf_verifier_log(env, "Unsupported version");

kernel/bpf/btf.c

5668

btf_verifier_log(env, "Unsupported flags");

kernel/bpf/btf.c

5673

btf_verifier_log(env, "No data");

kernel/bpf/btf.c

5677

return btf_check_sec_info(env, btf_data_size);

kernel/bpf/btf.c

5822

static int btf_check_type_tags(struct btf_verifier_env *env,

kernel/bpf/btf.c

5845

btf_verifier_log(env, "Max chain length or cycle detected");

kernel/bpf/btf.c

5850

btf_verifier_log(env, "Type tags don't precede modifiers");

kernel/bpf/btf.c

5889

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

5897

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

5898

if (!env)

kernel/bpf/btf.c

5904

err = bpf_vlog_init(&env->log, attr->btf_log_level,

kernel/bpf/btf.c

5914

env->btf = btf;

kernel/bpf/btf.c

5931

err = btf_parse_hdr(env);

kernel/bpf/btf.c

5937

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

5941

err = btf_parse_type_sec(env);

kernel/bpf/btf.c

5945

err = btf_check_type_tags(env, btf, 1);

kernel/bpf/btf.c

5949

struct_meta_tab = btf_parse_struct_metas(&env->log, btf);

kernel/bpf/btf.c

5966

err = finalize_log(&env->log, uattr, uattr_size);

kernel/bpf/btf.c

5970

btf_verifier_env_free(env);

kernel/bpf/btf.c

5978

ret = finalize_log(&env->log, uattr, uattr_size);

kernel/bpf/btf.c

5982

btf_verifier_env_free(env);

kernel/bpf/btf.c

6317

static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name,

kernel/bpf/btf.c

6331

env->btf = btf;

kernel/bpf/btf.c

6339

err = btf_parse_hdr(env);

kernel/bpf/btf.c

6345

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

6349

err = btf_check_all_metas(env);

kernel/bpf/btf.c

6353

err = btf_check_type_tags(env, btf, 1);

kernel/bpf/btf.c

6372

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

6377

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

6378

if (!env)

kernel/bpf/btf.c

6381

log = &env->log;

kernel/bpf/btf.c

6383

btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF);

kernel/bpf/btf.c

6395

btf_verifier_env_free(env);

kernel/bpf/btf.c

6417

struct btf_verifier_env *env = NULL;

kernel/bpf/btf.c

6427

env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN);

kernel/bpf/btf.c

6428

if (!env)

kernel/bpf/btf.c

6431

log = &env->log;

kernel/bpf/btf.c

6435

base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size);

kernel/bpf/btf.c

6449

env->btf = btf;

kernel/bpf/btf.c

6465

err = btf_parse_hdr(env);

kernel/bpf/btf.c

6471

err = btf_parse_str_sec(env);

kernel/bpf/btf.c

6475

err = btf_check_all_metas(env);

kernel/bpf/btf.c

6479

err = btf_check_type_tags(env, btf, btf_nr_types(base_btf));

kernel/bpf/btf.c

6491

btf_verifier_env_free(env);

kernel/bpf/btf.c

6497

btf_verifier_env_free(env);

kernel/bpf/btf.c

7787

int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)

kernel/bpf/btf.c

7789

bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL;

kernel/bpf/btf.c

7790

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/btf.c

7791

struct bpf_verifier_log *log = &env->log;

kernel/bpf/btf.c

7792

struct bpf_prog *prog = env->prog;

kernel/bpf/btf.c

7804

verifier_bug(env, "func_info undefined");

kernel/bpf/btf.c

7828

verifier_bug(env, "unreliable BTF for function %s()", tname);

kernel/bpf/liveness.c

139

static void compute_callchain(struct bpf_verifier_env *env, struct bpf_verifier_state *st,

kernel/bpf/liveness.c

142

struct bpf_subprog_info *subprog_info = env->subprog_info;

kernel/bpf/liveness.c

177

static struct func_instance *__lookup_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

180

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

190

subprog = bpf_find_containing_subprog(env, callchain->sp_starts[callchain->curframe]);

kernel/bpf/liveness.c

208

static struct func_instance *lookup_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

214

compute_callchain(env, st, &callchain, frameno);

kernel/bpf/liveness.c

215

return __lookup_instance(env, &callchain);

kernel/bpf/liveness.c

218

int bpf_stack_liveness_init(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

220

env->liveness = kvzalloc_obj(*env->liveness, GFP_KERNEL_ACCOUNT);

kernel/bpf/liveness.c

221

if (!env->liveness)

kernel/bpf/liveness.c

223

hash_init(env->liveness->func_instances);

kernel/bpf/liveness.c

227

void bpf_stack_liveness_free(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

233

if (!env->liveness)

kernel/bpf/liveness.c

235

hash_for_each_safe(env->liveness->func_instances, bkt, tmp, instance, hl_node) {

kernel/bpf/liveness.c

241

kvfree(env->liveness);

kernel/bpf/liveness.c

262

static struct per_frame_masks *alloc_frame_masks(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

278

void bpf_reset_live_stack_callchain(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

280

env->liveness->cur_instance = NULL;

kernel/bpf/liveness.c

284

static int ensure_cur_instance(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

286

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

292

instance = lookup_instance(env, env->cur_state, env->cur_state->curframe);

kernel/bpf/liveness.c

301

static int mark_stack_read(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

307

masks = alloc_frame_masks(env, instance, frame, insn_idx);

kernel/bpf/liveness.c

318

int bpf_mark_stack_read(struct bpf_verifier_env *env, u32 frame, u32 insn_idx, u64 mask)

kernel/bpf/liveness.c

322

err = ensure_cur_instance(env);

kernel/bpf/liveness.c

323

err = err ?: mark_stack_read(env, env->liveness->cur_instance, frame, insn_idx, mask);

kernel/bpf/liveness.c

327

static void reset_stack_write_marks(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

330

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

338

int bpf_reset_stack_write_marks(struct bpf_verifier_env *env, u32 insn_idx)

kernel/bpf/liveness.c

340

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

343

err = ensure_cur_instance(env);

kernel/bpf/liveness.c

347

reset_stack_write_marks(env, liveness->cur_instance, insn_idx);

kernel/bpf/liveness.c

351

void bpf_mark_stack_write(struct bpf_verifier_env *env, u32 frame, u64 mask)

kernel/bpf/liveness.c

353

env->liveness->write_masks_acc[frame] |= mask;

kernel/bpf/liveness.c

356

static int commit_stack_write_marks(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

359

struct bpf_liveness *liveness = env->liveness;

kernel/bpf/liveness.c

374

masks = alloc_frame_masks(env, instance, frame, liveness->write_insn_idx);

kernel/bpf/liveness.c

400

int bpf_commit_stack_write_marks(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

402

return commit_stack_write_marks(env, env->liveness->cur_instance);

kernel/bpf/liveness.c

405

static char *fmt_callchain(struct bpf_verifier_env *env, struct callchain *callchain)

kernel/bpf/liveness.c

407

char *buf_end = env->tmp_str_buf + sizeof(env->tmp_str_buf);

kernel/bpf/liveness.c

408

char *buf = env->tmp_str_buf;

kernel/bpf/liveness.c

415

return env->tmp_str_buf;

kernel/bpf/liveness.c

418

static void log_mask_change(struct bpf_verifier_env *env, struct callchain *callchain,

kernel/bpf/liveness.c

427

bpf_log(&env->log, "%s frame %d insn %d ", fmt_callchain(env, callchain), frame, insn_idx);

kernel/bpf/liveness.c

429

bpf_fmt_stack_mask(env->tmp_str_buf, sizeof(env->tmp_str_buf), new_ones);

kernel/bpf/liveness.c

430

bpf_log(&env->log, "+%s %s ", pfx, env->tmp_str_buf);

kernel/bpf/liveness.c

433

bpf_fmt_stack_mask(env->tmp_str_buf, sizeof(env->tmp_str_buf), new_zeros);

kernel/bpf/liveness.c

434

bpf_log(&env->log, "-%s %s", pfx, env->tmp_str_buf);

kernel/bpf/liveness.c

436

bpf_log(&env->log, "\n");

kernel/bpf/liveness.c

456

bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)

kernel/bpf/liveness.c

483

struct bpf_prog *prog = env->prog;

kernel/bpf/liveness.c

489

jt = env->insn_aux_data[idx].jt;

kernel/bpf/liveness.c

494

succ = env->succ;

kernel/bpf/liveness.c

510

static struct func_instance *get_outer_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

520

return __lookup_instance(env, &callchain);

kernel/bpf/liveness.c

532

static int propagate_to_outer_instance(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

542

outer_instance = get_outer_instance(env, instance);

kernel/bpf/liveness.c

547

reset_stack_write_marks(env, outer_instance, callsite);

kernel/bpf/liveness.c

552

bpf_mark_stack_write(env, frame, insn->must_write_acc);

kernel/bpf/liveness.c

553

err = mark_stack_read(env, outer_instance, frame, callsite, insn->live_before);

kernel/bpf/liveness.c

557

commit_stack_write_marks(env, outer_instance);

kernel/bpf/liveness.c

561

static inline bool update_insn(struct bpf_verifier_env *env,

kernel/bpf/liveness.c

564

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/liveness.c

571

succ = bpf_insn_successors(env, insn_idx);

kernel/bpf/liveness.c

597

if (unlikely(env->log.level & BPF_LOG_LEVEL2) &&

kernel/bpf/liveness.c

601

log_mask_change(env, &instance->callchain, "live",

kernel/bpf/liveness.c

603

log_mask_change(env, &instance->callchain, "written",

kernel/bpf/liveness.c

612

static int update_instance(struct bpf_verifier_env *env, struct func_instance *instance)

kernel/bpf/liveness.c

616

int *insn_postorder = env->cfg.insn_postorder;

kernel/bpf/liveness.c

639

subprog = bpf_find_containing_subprog(env, this_subprog_start);

kernel/bpf/liveness.c

652

changed |= update_insn(env, instance, frame, insn_postorder[i]);

kernel/bpf/liveness.c

656

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/liveness.c

657

bpf_log(&env->log, "%s live stack update done in %d iterations\n",

kernel/bpf/liveness.c

658

fmt_callchain(env, callchain), cnt);

kernel/bpf/liveness.c

662

err = propagate_to_outer_instance(env, instance);

kernel/bpf/liveness.c

678

int bpf_update_live_stack(struct bpf_verifier_env *env)

kernel/bpf/liveness.c

683

bpf_reset_live_stack_callchain(env);

kernel/bpf/liveness.c

684

for (frame = env->cur_state->curframe; frame >= 0; --frame) {

kernel/bpf/liveness.c

685

instance = lookup_instance(env, env->cur_state, frame);

kernel/bpf/liveness.c

690

err = update_instance(env, instance);

kernel/bpf/liveness.c

708

int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/liveness.c

710

struct live_stack_query *q = &env->liveness->live_stack_query;

kernel/bpf/liveness.c

716

instance = lookup_instance(env, st, frame);

kernel/bpf/liveness.c

726

bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi)

kernel/bpf/liveness.c

734

struct live_stack_query *q = &env->liveness->live_stack_query;

kernel/bpf/liveness.c

746

alive = bpf_calls_callback(env, callsite)

kernel/bpf/log.c

14

#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)

kernel/bpf/log.c

304

__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,

kernel/bpf/log.c

309

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/log.c

313

bpf_verifier_vlog(&env->log, fmt, args);

kernel/bpf/log.c

333

find_linfo(const struct bpf_verifier_env *env, u32 insn_off)

kernel/bpf/log.c

340

prog = env->prog;

kernel/bpf/log.c

381

__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,

kernel/bpf/log.c

389

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/log.c

392

prev_linfo = env->prev_linfo;

kernel/bpf/log.c

393

linfo = find_linfo(env, insn_off);

kernel/bpf/log.c

413

bpf_verifier_vlog(&env->log, prefix_fmt, args);

kernel/bpf/log.c

417

btf = env->prog->aux->btf;

kernel/bpf/log.c

419

verbose(env, "%s", s); /* source code line */

kernel/bpf/log.c

425

verbose(env, " @ %s:%u\n", fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col));

kernel/bpf/log.c

427

env->prev_linfo = linfo;

kernel/bpf/log.c

440

const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)

kernel/bpf/log.c

486

snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",

kernel/bpf/log.c

488

return env->tmp_str_buf;

kernel/bpf/log.c

562

static void verbose_unum(struct bpf_verifier_env *env, u64 num)

kernel/bpf/log.c

565

verbose(env, "%llu", num);

kernel/bpf/log.c

567

verbose(env, "%#llx", num);

kernel/bpf/log.c

570

static void verbose_snum(struct bpf_verifier_env *env, s64 num)

kernel/bpf/log.c

573

verbose(env, "%lld", num);

kernel/bpf/log.c

575

verbose(env, "%#llx", num);

kernel/bpf/log.c

591

static void print_scalar_ranges(struct bpf_verifier_env *env,

kernel/bpf/log.c

635

verbose(env, "%s%s=", *sep, m1->name);

kernel/bpf/log.c

646

verbose(env, "%s=", m2->name);

kernel/bpf/log.c

650

verbose_snum(env, m1->val);

kernel/bpf/log.c

652

verbose_unum(env, m1->val);

kernel/bpf/log.c

671

#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })

kernel/bpf/log.c

673

static void print_reg_state(struct bpf_verifier_env *env,

kernel/bpf/log.c

682

verbose(env, "P");

kernel/bpf/log.c

684

verbose_snum(env, reg->var_off.value);

kernel/bpf/log.c

688

verbose(env, "%s", reg_type_str(env, t));

kernel/bpf/log.c

693

verbose(env, "[%d]", reg->frameno);

kernel/bpf/log.c

695

verbose_snum(env, reg->var_off.value + reg->off);

kernel/bpf/log.c

700

verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));

kernel/bpf/log.c

701

verbose(env, "(");

kernel/bpf/log.c

705

verbose(env, "%+d", reg->off);

kernel/bpf/log.c

719

verbose_snum(env, reg->off);

kernel/bpf/log.c

723

verbose_unum(env, reg->range);

kernel/bpf/log.c

727

verbose_unum(env, reg->mem_size);

kernel/bpf/log.c

735

verbose_snum(env, reg->var_off.value);

kernel/bpf/log.c

738

print_scalar_ranges(env, reg, &sep);

kernel/bpf/log.c

746

verbose(env, ")");

kernel/bpf/log.c

749

void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

kernel/bpf/log.c

757

verbose(env, " frame%d:", state->frameno);

kernel/bpf/log.c

762

if (!print_all && !reg_scratched(env, i))

kernel/bpf/log.c

764

verbose(env, " R%d", i);

kernel/bpf/log.c

765

verbose(env, "=");

kernel/bpf/log.c

766

print_reg_state(env, state, reg);

kernel/bpf/log.c

775

if (!print_all && !stack_slot_scratched(env, i))

kernel/bpf/log.c

797

verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);

kernel/bpf/log.c

798

print_reg_state(env, state, reg);

kernel/bpf/log.c

805

verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);

kernel/bpf/log.c

806

verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type));

kernel/bpf/log.c

813

verbose(env, ")");

kernel/bpf/log.c

820

verbose(env, " fp%d=iter_%s(ref_id=%d,state=%s,depth=%u)",

kernel/bpf/log.c

829

verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);

kernel/bpf/log.c

834

verbose(env, " refs=%d", vstate->refs[0].id);

kernel/bpf/log.c

837

verbose(env, ",%d", vstate->refs[i].id);

kernel/bpf/log.c

840

verbose(env, " cb");

kernel/bpf/log.c

842

verbose(env, " async_cb");

kernel/bpf/log.c

843

verbose(env, "\n");

kernel/bpf/log.c

845

mark_verifier_state_clean(env);

kernel/bpf/log.c

854

void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,

kernel/bpf/log.c

857

if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {

kernel/bpf/log.c

859

bpf_vlog_reset(&env->log, env->prev_log_pos - 1);

kernel/bpf/log.c

860

verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' ');

kernel/bpf/log.c

862

verbose(env, "%d:", env->insn_idx);

kernel/bpf/log.c

864

print_verifier_state(env, vstate, frameno, false);

kernel/bpf/offload.c

303

int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,

kernel/bpf/offload.c

310

offload = env->prog->aux->offload;

kernel/bpf/offload.c

312

ret = offload->offdev->ops->insn_hook(env, insn_idx,

kernel/bpf/offload.c

319

int bpf_prog_offload_finalize(struct bpf_verifier_env *env)

kernel/bpf/offload.c

325

offload = env->prog->aux->offload;

kernel/bpf/offload.c

328

ret = offload->offdev->ops->finalize(env);

kernel/bpf/offload.c

338

bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,

kernel/bpf/offload.c

346

offload = env->prog->aux->offload;

kernel/bpf/offload.c

350

ret = ops->replace_insn(env, off, insn);

kernel/bpf/offload.c

357

bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

kernel/bpf/offload.c

363

offload = env->prog->aux->offload;

kernel/bpf/offload.c

366

ret = offload->offdev->ops->remove_insns(env, off, cnt);

kernel/bpf/verifier.c

10002

verifier_bug(env, "invalid map_ptr to access map->key");

kernel/bpf/verifier.c

10006

err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL);

kernel/bpf/verifier.c

10010

err = get_constant_map_key(env, reg, key_size, &meta->const_map_key);

kernel/bpf/verifier.c

10029

verifier_bug(env, "invalid map_ptr to access map->value");

kernel/bpf/verifier.c

10033

err = check_helper_mem_access(env, regno, meta->map.ptr->value_size,

kernel/bpf/verifier.c

10039

verbose(env, "Helper has invalid btf_id in R%d\n", regno);

kernel/bpf/verifier.c

10046

if (in_rbtree_lock_required_cb(env)) {

kernel/bpf/verifier.c

10047

verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");

kernel/bpf/verifier.c

10051

err = process_spin_lock(env, regno, PROCESS_SPIN_LOCK);

kernel/bpf/verifier.c

10055

err = process_spin_lock(env, regno, 0);

kernel/bpf/verifier.c

10059

verifier_bug(env, "spin lock arg on unexpected helper");

kernel/bpf/verifier.c

1006

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

10064

err = process_timer_helper(env, regno, meta);

kernel/bpf/verifier.c

10077

err = check_helper_mem_access(env, regno, fn->arg_size[arg],

kernel/bpf/verifier.c

10083

err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);

kernel/bpf/verifier.c

10087

err = check_mem_size_reg(env, reg, regno,

kernel/bpf/verifier.c

10093

err = check_mem_size_reg(env, reg, regno,

kernel/bpf/verifier.c

10099

err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);

kernel/bpf/verifier.c

10105

verbose(env, "R%d is not a known constant'\n",

kernel/bpf/verifier.c

10110

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

10116

err = check_reg_const_str(env, reg, regno);

kernel/bpf/verifier.c

10122

err = process_kptr_func(env, regno, meta);

kernel/bpf/verifier.c

10131

static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)

kernel/bpf/verifier.c

10133

enum bpf_attach_type eatype = env->prog->expected_attach_type;

kernel/bpf/verifier.c

10134

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

10165

verbose(env, "cannot update sockmap in this context\n");

kernel/bpf/verifier.c

10169

static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10171

return env->prog->jit_requested &&

kernel/bpf/verifier.c

10175

static int check_map_func_compatibility(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1021

static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

1024

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

10251

!may_update_sockmap(env, func_id))

kernel/bpf/verifier.c

10260

!may_update_sockmap(env, func_id))

kernel/bpf/verifier.c

10314

if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {

kernel/bpf/verifier.c

10315

verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");

kernel/bpf/verifier.c

1036

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

10423

verbose(env, "cannot pass map_type %d into func %s#%d\n",

kernel/bpf/verifier.c

1045

static bool in_rcu_cs(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

1049

static int mark_stack_slots_iter(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10532

static void clear_all_pkt_pointers(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

10537

bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({

kernel/bpf/verifier.c

10539

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

1054

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

1057

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

10589

static int release_reference(struct bpf_verifier_env *env, int ref_obj_id)

kernel/bpf/verifier.c

10591

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

10602

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

10608

static void invalidate_non_owning_refs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

1061

id = acquire_reference(env, insn_idx);

kernel/bpf/verifier.c

10613

bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({

kernel/bpf/verifier.c

10615

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

10619

static void clear_caller_saved_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10626

mark_reg_not_init(env, regs, caller_saved[i]);

kernel/bpf/verifier.c

10627

__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

10631

typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10636

static int set_callee_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10640

static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,

kernel/bpf/verifier.c

10648

verbose(env, "the call stack of %d frames is too deep\n",

kernel/bpf/verifier.c

10654

verifier_bug(env, "Frame %d already allocated", state->curframe + 1);

kernel/bpf/verifier.c

10668

init_func_state(env, callee,

kernel/bpf/verifier.c

10673

err = set_callee_state_cb(env, caller, callee, callsite);

kernel/bpf/verifier.c

10688

static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

10692

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

10693

struct bpf_verifier_log *log = &env->log;

kernel/bpf/verifier.c

10697

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

1072

if (in_rcu_cs(env))

kernel/bpf/verifier.c

10721

ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);

kernel/bpf/verifier.c

10732

ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);

kernel/bpf/verifier.c

10735

if (check_mem_reg(env, reg, regno, arg->mem_size))

kernel/bpf/verifier.c

10754

ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);

kernel/bpf/verifier.c

10758

ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);

kernel/bpf/verifier.c

10769

err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);

kernel/bpf/verifier.c

10770

err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);

kernel/bpf/verifier.c

10774

verifier_bug(env, "unrecognized arg#%d type %d", i, arg->arg_type);

kernel/bpf/verifier.c

10789

static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,

kernel/bpf/verifier.c

10792

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

10807

err = btf_check_func_arg_match(env, subprog, btf, regs);

kernel/bpf/verifier.c

10817

static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

10821

struct bpf_verifier_state *state = env->cur_state, *callback_state;

kernel/bpf/verifier.c

10826

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

10834

env->subprog_info[subprog].is_cb = true;

kernel/bpf/verifier.c

10837

verifier_bug(env, "kfunc %s#%d not marked as callback-calling",

kernel/bpf/verifier.c

10842

verifier_bug(env, "helper %s#%d not marked as callback-calling",

kernel/bpf/verifier.c

10851

env->subprog_info[subprog].is_async_cb = true;

kernel/bpf/verifier.c

10852

async_cb = push_async_cb(env, env->subprog_info[subprog].start,

kernel/bpf/verifier.c

10854

is_async_cb_sleepable(env, insn));

kernel/bpf/verifier.c

1086

bpf_mark_stack_write(env, state->frameno, BIT(spi - i));

kernel/bpf/verifier.c

10861

err = set_callee_state_cb(env, caller, callee, insn_idx);

kernel/bpf/verifier.c

1087

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

10871

callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);

kernel/bpf/verifier.c

10875

err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,

kernel/bpf/verifier.c

10886

static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

10889

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

10894

subprog = find_subprog(env, target_insn);

kernel/bpf/verifier.c

10895

if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program",

kernel/bpf/verifier.c

10900

err = btf_check_subprog_call(env, subprog, caller->regs);

kernel/bpf/verifier.c

10903

if (subprog_is_global(env, subprog)) {

kernel/bpf/verifier.c

10904

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

10906

if (env->cur_state->active_locks) {

kernel/bpf/verifier.c

10907

verbose(env, "global function calls are not allowed while holding a lock,\n"

kernel/bpf/verifier.c

10912

if (env->subprog_info[subprog].might_sleep &&

kernel/bpf/verifier.c

10913

(env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||

kernel/bpf/verifier.c

10914

env->cur_state->active_irq_id || !in_sleepable(env))) {

kernel/bpf/verifier.c

10915

verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"

kernel/bpf/verifier.c

10922

verbose(env, "Caller passes invalid args into func#%d ('%s')\n",

kernel/bpf/verifier.c

10927

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

10928

verbose(env, "Func#%d ('%s') is global and assumed valid.\n",

kernel/bpf/verifier.c

1093

static int unmark_stack_slots_iter(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

10930

if (env->subprog_info[subprog].changes_pkt_data)

kernel/bpf/verifier.c

10931

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

10933

subprog_aux(env, subprog)->called = true;

kernel/bpf/verifier.c

10934

clear_caller_saved_regs(env, caller->regs);

kernel/bpf/verifier.c

10937

mark_reg_unknown(env, caller->regs, BPF_REG_0);

kernel/bpf/verifier.c

10947

err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);

kernel/bpf/verifier.c

10951

clear_caller_saved_regs(env, caller->regs);

kernel/bpf/verifier.c

10954

*insn_idx = env->subprog_info[subprog].start - 1;

kernel/bpf/verifier.c

10956

bpf_reset_live_stack_callchain(env);

kernel/bpf/verifier.c

10958

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

10959

verbose(env, "caller:\n");

kernel/bpf/verifier.c

1096

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

10960

print_verifier_state(env, state, caller->frameno, true);

kernel/bpf/verifier.c

10961

verbose(env, "callee:\n");

kernel/bpf/verifier.c

10962

print_verifier_state(env, state, state->curframe, true);

kernel/bpf/verifier.c

10968

int map_set_for_each_callback_args(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1099

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

10991

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

10995

static int set_callee_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11009

static int set_map_elem_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11014

struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

11022

verbose(env, "callback function not allowed for map\n");

kernel/bpf/verifier.c

11026

err = map->ops->map_set_for_each_callback_args(env, caller, callee);

kernel/bpf/verifier.c

11035

static int set_loop_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11048

__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

11049

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11050

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11057

static int set_timer_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1108

WARN_ON_ONCE(release_reference(env, st->ref_obj_id));

kernel/bpf/verifier.c

11080

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11081

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11087

static int set_find_vma_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1110

__mark_reg_not_init(env, st);

kernel/bpf/verifier.c

11108

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11109

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11115

static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11124

__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);

kernel/bpf/verifier.c

11125

mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);

kernel/bpf/verifier.c

11129

__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

11130

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11131

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11138

static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1115

bpf_mark_stack_write(env, state->frameno, BIT(spi - i));

kernel/bpf/verifier.c

11158

ref_set_non_owning(env, &callee->regs[BPF_REG_1]);

kernel/bpf/verifier.c

1116

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

11160

ref_set_non_owning(env, &callee->regs[BPF_REG_2]);

kernel/bpf/verifier.c

11162

__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);

kernel/bpf/verifier.c

11163

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11164

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11170

static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11193

__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);

kernel/bpf/verifier.c

11194

__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

kernel/bpf/verifier.c

11206

static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11208

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

11209

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

1122

static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11234

static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)

kernel/bpf/verifier.c

11236

struct bpf_verifier_state *state = env->cur_state, *prev_st;

kernel/bpf/verifier.c

11242

err = bpf_update_live_stack(env);

kernel/bpf/verifier.c

1125

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

11255

verbose(env, "cannot return stack pointer to the caller\n");

kernel/bpf/verifier.c

11262

verbose(env, "R0 not a scalar value\n");

kernel/bpf/verifier.c

11267

err = mark_chain_precision(env, BPF_REG_0);

kernel/bpf/verifier.c

11273

verbose_invalid_scalar(env, r0, callee->callback_ret_range,

kernel/bpf/verifier.c

11277

if (!bpf_calls_callback(env, callee->callsite)) {

kernel/bpf/verifier.c

11278

verifier_bug(env, "in callback at %d, callsite %d !calls_callback",

kernel/bpf/verifier.c

11297

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

11298

verbose(env, "returning from callee:\n");

kernel/bpf/verifier.c

11299

print_verifier_state(env, state, callee->frameno, true);

kernel/bpf/verifier.c

11300

verbose(env, "to caller at %d:\n", *insn_idx);

kernel/bpf/verifier.c

11301

print_verifier_state(env, state, caller->frameno, true);

kernel/bpf/verifier.c

11319

prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;

kernel/bpf/verifier.c

1132

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

11321

err = widen_imprecise_scalars(env, prev_st, state);

kernel/bpf/verifier.c

11328

static int do_refine_retval_range(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11363

return reg_bounds_sanity_check(env, ret_reg, "retval");

kernel/bpf/verifier.c

11367

record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,

kernel/bpf/verifier.c

11370

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

11386

verifier_bug(env, "expected map for helper call");

kernel/bpf/verifier.c

11399

verbose(env, "write into map forbidden\n");

kernel/bpf/verifier.c

11413

record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,

kernel/bpf/verifier.c

11416

struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

11425

verbose(env, "expected prog array map for tail call");

kernel/bpf/verifier.c

11429

reg = reg_state(env, BPF_REG_3);

kernel/bpf/verifier.c

11438

err = mark_chain_precision(env, BPF_REG_3);

kernel/bpf/verifier.c

11449

static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)

kernel/bpf/verifier.c

11451

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

11452

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

11453

struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);

kernel/bpf/verifier.c

11457

if (!exception_exit && cur_func(env)->frameno)

kernel/bpf/verifier.c

11469

verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",

kernel/bpf/verifier.c

11476

static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)

kernel/bpf/verifier.c

11480

if (check_lock && env->cur_state->active_locks) {

kernel/bpf/verifier.c

11481

verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);

kernel/bpf/verifier.c

11485

err = check_reference_leak(env, exception_exit);

kernel/bpf/verifier.c

11487

verbose(env, "%s would lead to reference leak\n", prefix);

kernel/bpf/verifier.c

1149

static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

11491

if (check_lock && env->cur_state->active_irq_id) {

kernel/bpf/verifier.c

11492

verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix);

kernel/bpf/verifier.c

11496

if (check_lock && env->cur_state->active_rcu_locks) {

kernel/bpf/verifier.c

11497

verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);

kernel/bpf/verifier.c

11501

if (check_lock && env->cur_state->active_preempt_locks) {

kernel/bpf/verifier.c

11502

verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);

kernel/bpf/verifier.c

11509

static int check_bpf_snprintf_call(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1152

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

11532

verbose(env, "failed to retrieve map value address\n");

kernel/bpf/verifier.c

11542

verbose(env, "Invalid format string\n");

kernel/bpf/verifier.c

11547

static int check_get_func_ip(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11549

enum bpf_prog_type type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

1155

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

11553

if (!bpf_prog_has_trampoline(env->prog)) {

kernel/bpf/verifier.c

11554

verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",

kernel/bpf/verifier.c

11563

verbose(env, "func %s#%d not supported for program type %d\n",

kernel/bpf/verifier.c

11568

static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11570

return &env->insn_aux_data[env->insn_idx];

kernel/bpf/verifier.c

11573

static bool loop_flag_is_zero(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11575

struct bpf_reg_state *reg = reg_state(env, BPF_REG_4);

kernel/bpf/verifier.c

11579

mark_chain_precision(env, BPF_REG_4);

kernel/bpf/verifier.c

11584

static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)

kernel/bpf/verifier.c

11586

struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;

kernel/bpf/verifier.c

11590

state->fit_for_inline = loop_flag_is_zero(env);

kernel/bpf/verifier.c

11598

state->fit_for_inline = (loop_flag_is_zero(env) &&

kernel/bpf/verifier.c

11617

static int get_helper_proto(struct bpf_verifier_env *env, int func_id,

kernel/bpf/verifier.c

11623

if (!env->ops->get_func_proto)

kernel/bpf/verifier.c

11626

*ptr = env->ops->get_func_proto(func_id, env->prog);

kernel/bpf/verifier.c

11631

static inline bool in_sleepable_context(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

11633

return !env->cur_state->active_rcu_locks &&

kernel/bpf/verifier.c

11634

!env->cur_state->active_preempt_locks &&

kernel/bpf/verifier.c

11635

!env->cur_state->active_locks &&

kernel/bpf/verifier.c

11636

!env->cur_state->active_irq_id &&

kernel/bpf/verifier.c

11637

in_sleepable(env);

kernel/bpf/verifier.c

11640

static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

11643

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

11656

err = get_helper_proto(env, insn->imm, &fn);

kernel/bpf/verifier.c

11658

verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

11663

verbose(env, "program of this type cannot use helper %s#%d\n",

kernel/bpf/verifier.c

11669

if (!env->prog->gpl_compatible && fn->gpl_only) {

kernel/bpf/verifier.c

11670

verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");

kernel/bpf/verifier.c

11674

if (fn->allowed && !fn->allowed(env->prog)) {

kernel/bpf/verifier.c

11675

verbose(env, "helper call is not allowed in probe\n");

kernel/bpf/verifier.c

11679

if (!in_sleepable(env) && fn->might_sleep) {

kernel/bpf/verifier.c

11680

verbose(env, "helper call might sleep in a non-sleepable prog\n");

kernel/bpf/verifier.c

11687

verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

11696

verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id);

kernel/bpf/verifier.c

11700

if (env->cur_state->active_rcu_locks) {

kernel/bpf/verifier.c

11702

verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",

kernel/bpf/verifier.c

11708

if (env->cur_state->active_preempt_locks) {

kernel/bpf/verifier.c

11710

verbose(env, "sleepable helper %s#%d in non-preemptible region\n",

kernel/bpf/verifier.c

11716

if (env->cur_state->active_irq_id) {

kernel/bpf/verifier.c

11718

verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n",

kernel/bpf/verifier.c

11725

if (!in_sleepable_context(env))

kernel/bpf/verifier.c

11726

env->insn_aux_data[insn_idx].non_sleepable = true;

kernel/bpf/verifier.c

11731

err = check_func_arg(env, i, &meta, fn, insn_idx);

kernel/bpf/verifier.c

11736

err = record_func_map(env, &meta, func_id, insn_idx);

kernel/bpf/verifier.c

11740

err = record_func_key(env, &meta, func_id, insn_idx);

kernel/bpf/verifier.c

11748

err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,

kernel/bpf/verifier.c

11754

regs = cur_regs(env);

kernel/bpf/verifier.c

11759

err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);

kernel/bpf/verifier.c

11762

bool in_rcu = in_rcu_cs(env);

kernel/bpf/verifier.c

11766

err = release_reference_nomark(env->cur_state, ref_obj_id);

kernel/bpf/verifier.c

11768

bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({

kernel/bpf/verifier.c

11775

mark_reg_invalid(env, reg);

kernel/bpf/verifier.c

11781

err = release_reference(env, meta.ref_obj_id);

kernel/bpf/verifier.c

11789

verbose(env, "func %s#%d reference has not been acquired before\n",

kernel/bpf/verifier.c

11797

err = check_resource_leak(env, false, true, "tail_call");

kernel/bpf/verifier.c

11806

verbose(env, "get_local_storage() doesn't support non-zero flags\n");

kernel/bpf/verifier.c

1181

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);

kernel/bpf/verifier.c

11811

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

11815

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

11819

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

11823

err = check_bpf_snprintf_call(env, regs);

kernel/bpf/verifier.c

11826

update_loop_inline_state(env, meta.subprogno);

kernel/bpf/verifier.c

11830

err = mark_chain_precision(env, BPF_REG_1);

kernel/bpf/verifier.c

11833

if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {

kernel/bpf/verifier.c

11834

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

11837

cur_func(env)->callback_depth = 0;

kernel/bpf/verifier.c

11838

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

11839

verbose(env, "frame%d bpf_loop iteration limit reached\n",

kernel/bpf/verifier.c

1184

static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

11840

env->cur_state->curframe);

kernel/bpf/verifier.c

11845

verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",

kernel/bpf/verifier.c

11846

reg_type_str(env, regs[BPF_REG_1].type));

kernel/bpf/verifier.c

11852

env->prog->expected_attach_type == BPF_LSM_CGROUP) {

kernel/bpf/verifier.c

11853

if (!env->prog->aux->attach_func_proto->type) {

kernel/bpf/verifier.c

11857

verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");

kernel/bpf/verifier.c

11867

reg = get_dynptr_arg_reg(env, fn, regs);

kernel/bpf/verifier.c

11873

verifier_bug(env, "meta.dynptr_id already set");

kernel/bpf/verifier.c

11877

verifier_bug(env, "meta.ref_obj_id already set");

kernel/bpf/verifier.c

11881

id = dynptr_id(env, reg);

kernel/bpf/verifier.c

11883

verifier_bug(env, "failed to obtain dynptr id");

kernel/bpf/verifier.c

11887

ref_obj_id = dynptr_ref_obj_id(env, reg);

kernel/bpf/verifier.c

11889

verifier_bug(env, "failed to obtain dynptr ref_obj_id");

kernel/bpf/verifier.c

1189

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

11903

reg = get_dynptr_arg_reg(env, fn, regs);

kernel/bpf/verifier.c

11907

dynptr_type = dynptr_get_type(env, reg);

kernel/bpf/verifier.c

11929

verbose(env, "Helper has invalid btf/btf_id in R1\n");

kernel/bpf/verifier.c

11933

env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;

kernel/bpf/verifier.c

11938

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

1194

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

11948

mark_reg_not_init(env, regs, caller_saved[i]);

kernel/bpf/verifier.c

11949

check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

11962

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

11969

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

11975

verifier_bug(env, "unexpected null map_ptr");

kernel/bpf/verifier.c

1198

id = acquire_irq_state(env, insn_idx);

kernel/bpf/verifier.c

11990

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

11994

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

11998

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12002

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12006

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12014

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12025

verbose(env, "unable to resolve the size of type '%s': %ld\n",

kernel/bpf/verifier.c

1205

bpf_mark_stack_write(env, reg->frameno, BIT(spi));

kernel/bpf/verifier.c

12054

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12066

verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type",

kernel/bpf/verifier.c

12074

verbose(env, "invalid return type %u of func %s#%d\n",

kernel/bpf/verifier.c

12084

verbose(env, "unknown return type %u of func %s#%d\n",

kernel/bpf/verifier.c

12090

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

12093

verifier_bug(env, "func %s#%d sets ref_obj_id more than once",

kernel/bpf/verifier.c

12105

int id = acquire_reference(env, insn_idx);

kernel/bpf/verifier.c

12115

err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);

kernel/bpf/verifier.c

12119

err = check_map_func_compatibility(env, meta.map.ptr, func_id);

kernel/bpf/verifier.c

12125

!env->prog->has_callchain_buf) {

kernel/bpf/verifier.c

12136

verbose(env, err_str, func_id_name(func_id), func_id);

kernel/bpf/verifier.c

1214

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

12140

env->prog->has_callchain_buf = true;

kernel/bpf/verifier.c

12144

env->prog->call_get_stack = true;

kernel/bpf/verifier.c

12147

if (check_get_func_ip(env))

kernel/bpf/verifier.c

12149

env->prog->call_get_func_ip = true;

kernel/bpf/verifier.c

12153

if (env->cur_state->curframe) {

kernel/bpf/verifier.c

12156

mark_reg_scratched(env, BPF_REG_0);

kernel/bpf/verifier.c

12157

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

12160

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

12161

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

12162

err = prepare_func_exit(env, &env->insn_idx);

kernel/bpf/verifier.c

12165

env->insn_idx--;

kernel/bpf/verifier.c

12172

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

12179

static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs,

kernel/bpf/verifier.c

1218

static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

12187

DEF_NOT_SUBREG : env->insn_idx + 1;

kernel/bpf/verifier.c

12190

mark_insn_zext(env, reg);

kernel/bpf/verifier.c

12194

static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

12197

return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size);

kernel/bpf/verifier.c

1221

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

1226

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

1237

verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n",

kernel/bpf/verifier.c

12417

static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,

kernel/bpf/verifier.c

1242

err = release_irq_state(env->cur_state, st->ref_obj_id);

kernel/bpf/verifier.c

12435

static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

12452

verbose(env, "max struct nesting depth exceeded\n");

kernel/bpf/verifier.c

12455

if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))

kernel/bpf/verifier.c

1247

for (int i = 0; i < env->cur_state->acquired_refs; i++) {

kernel/bpf/verifier.c

1248

if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) {

kernel/bpf/verifier.c

1249

insn_idx = env->cur_state->refs[i].insn_idx;

kernel/bpf/verifier.c

1254

verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n",

kernel/bpf/verifier.c

1255

env->cur_state->active_irq_id, insn_idx);

kernel/bpf/verifier.c

1259

__mark_reg_not_init(env, st);

kernel/bpf/verifier.c

1261

bpf_mark_stack_write(env, reg->frameno, BIT(spi));

kernel/bpf/verifier.c

1266

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

12679

get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

12686

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

1270

static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

12705

if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))

kernel/bpf/verifier.c

1272

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

12759

verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",

kernel/bpf/verifier.c

12766

if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))

kernel/bpf/verifier.c

12774

if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&

kernel/bpf/verifier.c

12776

verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",

kernel/bpf/verifier.c

12783

static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1280

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

12831

btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))

kernel/bpf/verifier.c

12840

struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match);

kernel/bpf/verifier.c

12847

verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",

kernel/bpf/verifier.c

12855

static int process_irq_flag(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

12858

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

12873

verifier_bug(env, "unknown irq flags kfunc");

kernel/bpf/verifier.c

12878

if (!is_irq_flag_reg_valid_uninit(env, reg)) {

kernel/bpf/verifier.c

12879

verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1);

kernel/bpf/verifier.c

12883

err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false);

kernel/bpf/verifier.c

12887

err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class);

kernel/bpf/verifier.c

12891

err = is_irq_flag_reg_valid_init(env, reg);

kernel/bpf/verifier.c

12893

verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1);

kernel/bpf/verifier.c

12897

err = mark_irq_flag_read(env, reg);

kernel/bpf/verifier.c

12901

err = unmark_stack_slot_irq_flag(env, reg, kfunc_class);

kernel/bpf/verifier.c

12909

static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

12913

if (!env->cur_state->active_locks) {

kernel/bpf/verifier.c

12914

verifier_bug(env, "%s w/o active lock", __func__);

kernel/bpf/verifier.c

12919

verifier_bug(env, "NON_OWN_REF already set");

kernel/bpf/verifier.c

12930

static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)

kernel/bpf/verifier.c

12932

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

12938

verifier_bug(env, "ref_obj_id is zero for owning -> non-owning conversion");

kernel/bpf/verifier.c

1294

static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

12949

bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({

kernel/bpf/verifier.c

12952

ref_set_non_owning(env, reg);

kernel/bpf/verifier.c

12958

verifier_bug(env, "ref state missing for ref_obj_id");

kernel/bpf/verifier.c

1296

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

13006

static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

1301

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

13020

verifier_bug(env, "unknown reg type for lock check");

kernel/bpf/verifier.c

13025

if (!env->cur_state->active_locks)

kernel/bpf/verifier.c

13027

s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr);

kernel/bpf/verifier.c

13029

verbose(env, "held lock and object are not in the same allocation\n");

kernel/bpf/verifier.c

13129

static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13143

verbose(env, "verifier internal error: unexpected graph root argument type %s\n",

kernel/bpf/verifier.c

13149

verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",

kernel/bpf/verifier.c

13154

static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13172

verbose(env, "verifier internal error: unexpected graph node argument type %s\n",

kernel/bpf/verifier.c

13178

verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",

kernel/bpf/verifier.c

13184

__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13196

verifier_bug(env, "unexpected btf mismatch in kfunc call");

kernel/bpf/verifier.c

13200

if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))

kernel/bpf/verifier.c

13205

verbose(env,

kernel/bpf/verifier.c

13215

verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);

kernel/bpf/verifier.c

13220

if (check_reg_allocation_locked(env, reg)) {

kernel/bpf/verifier.c

13221

verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",

kernel/bpf/verifier.c

13227

verifier_bug(env, "repeating %s arg", head_type_name);

kernel/bpf/verifier.c

13234

static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13238

return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,

kernel/bpf/verifier.c

13242

static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13246

return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,

kernel/bpf/verifier.c

13251

__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13264

verifier_bug(env, "unexpected btf mismatch in kfunc call");

kernel/bpf/verifier.c

13268

if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))

kernel/bpf/verifier.c

13273

verbose(env,

kernel/bpf/verifier.c

13282

verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);

kernel/bpf/verifier.c

13290

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,

kernel/bpf/verifier.c

13292

verbose(env, "operation on %s expects arg#1 %s at offset=%d "

kernel/bpf/verifier.c

13305

verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",

kernel/bpf/verifier.c

13315

static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13319

return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,

kernel/bpf/verifier.c

13324

static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13328

return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,

kernel/bpf/verifier.c

13339

static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

13341

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

13347

if (env->prog->expected_attach_type == BPF_TRACE_ITER)

kernel/bpf/verifier.c

13351

return in_sleepable(env);

kernel/bpf/verifier.c

13355

static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,

kernel/bpf/verifier.c

13368

verbose(env, "Function %s has %d > %d args\n", func_name, nargs,

kernel/bpf/verifier.c

13377

struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];

kernel/bpf/verifier.c

13392

verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc");

kernel/bpf/verifier.c

13396

cur_aux(env)->arg_prog = regno;

kernel/bpf/verifier.c

13402

verbose(env, "R%d is not a scalar\n", regno);

kernel/bpf/verifier.c

13408

verifier_bug(env, "only one constant argument permitted");

kernel/bpf/verifier.c

13412

verbose(env, "R%d must be a known constant\n", regno);

kernel/bpf/verifier.c

13415

ret = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

13429

verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");

kernel/bpf/verifier.c

13434

verbose(env, "R%d is not a const\n", regno);

kernel/bpf/verifier.c

13439

ret = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

13447

verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));

kernel/bpf/verifier.c

13453

verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);

kernel/bpf/verifier.c

13459

verifier_bug(env, "more than one arg with ref_obj_id R%d %u %u",

kernel/bpf/verifier.c

13472

kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);

kernel/bpf/verifier.c

13481

verbose(env, "pointer in R%d isn't map pointer\n", regno);

kernel/bpf/verifier.c

13501

verbose(env,

kernel/bpf/verifier.c

13506

verbose(env,

kernel/bpf/verifier.c

13519

verbose(env, "R%d must be referenced or trusted\n", regno);

kernel/bpf/verifier.c

13523

verbose(env, "R%d must be a rcu pointer\n", regno);

kernel/bpf/verifier.c

13547

verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);

kernel/bpf/verifier.c

13553

ret = check_func_arg_reg_off(env, reg, regno, arg_type);

kernel/bpf/verifier.c

13560

verbose(env, "arg#%d expected pointer to ctx, but got %s\n",

kernel/bpf/verifier.c

13561

i, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

13566

ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));

kernel/bpf/verifier.c

13575

verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i);

kernel/bpf/verifier.c

13580

verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i);

kernel/bpf/verifier.c

13584

verbose(env, "arg#%d expected pointer to allocated object\n", i);

kernel/bpf/verifier.c

13588

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

13623

verifier_bug(env, "no dynptr type for parent of clone");

kernel/bpf/verifier.c

13630

verifier_bug(env, "missing ref obj id for parent of clone");

kernel/bpf/verifier.c

13635

ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);

kernel/bpf/verifier.c

13640

int id = dynptr_id(env, reg);

kernel/bpf/verifier.c

13643

verifier_bug(env, "failed to obtain dynptr id");

kernel/bpf/verifier.c

13647

meta->initialized_dynptr.type = dynptr_get_type(env, reg);

kernel/bpf/verifier.c

13648

meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);

kernel/bpf/verifier.c

13655

if (!check_css_task_iter_allowlist(env)) {

kernel/bpf/verifier.c

13656

verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");

kernel/bpf/verifier.c

13660

ret = process_iter_arg(env, regno, insn_idx, meta);

kernel/bpf/verifier.c

13667

verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);

kernel/bpf/verifier.c

13671

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

13674

ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);

kernel/bpf/verifier.c

13681

verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);

kernel/bpf/verifier.c

13685

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

13688

ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);

kernel/bpf/verifier.c

13694

verbose(env, "arg#%d expected pointer to allocated object\n", i);

kernel/bpf/verifier.c

13698

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

13701

ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);

kernel/bpf/verifier.c

13708

verbose(env, "arg#%d expected pointer to allocated object\n", i);

kernel/bpf/verifier.c

13712

verbose(env, "allocated object must be referenced\n");

kernel/bpf/verifier.c

13717

verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);

kernel/bpf/verifier.c

1372

static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)

kernel/bpf/verifier.c

13720

if (in_rbtree_lock_required_cb(env)) {

kernel/bpf/verifier.c

13721

verbose(env, "%s not allowed in rbtree cb\n", func_name);

kernel/bpf/verifier.c

13726

ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);

kernel/bpf/verifier.c

13741

verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

13742

verbose(env, "expected %s or socket\n",

kernel/bpf/verifier.c

13743

reg_type_str(env, base_type(reg->type) |

kernel/bpf/verifier.c

13747

ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);

kernel/bpf/verifier.c

13754

verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",

kernel/bpf/verifier.c

13758

ret = check_mem_reg(env, reg, regno, type_size);

kernel/bpf/verifier.c

13770

ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);

kernel/bpf/verifier.c

13772

verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);

kernel/bpf/verifier.c

13779

verifier_bug(env, "only one constant argument permitted");

kernel/bpf/verifier.c

13783

verbose(env, "R%d must be a known constant\n", regno + 1);

kernel/bpf/verifier.c

13796

verbose(env, "arg%d expected pointer to func\n", i);

kernel/bpf/verifier.c

13803

verbose(env, "arg#%d is neither owning or non-owning ref\n", i);

kernel/bpf/verifier.c

13811

verifier_bug(env, "Couldn't find btf_record");

kernel/bpf/verifier.c

13816

verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);

kernel/bpf/verifier.c

13825

verbose(env, "arg#%d doesn't point to a const string\n", i);

kernel/bpf/verifier.c

13828

ret = check_reg_const_str(env, reg, regno);

kernel/bpf/verifier.c

13834

verbose(env, "arg#%d doesn't point to a map value\n", i);

kernel/bpf/verifier.c

13837

ret = check_map_field_pointer(env, regno, BPF_WORKQUEUE, &meta->map);

kernel/bpf/verifier.c

13843

verbose(env, "arg#%d doesn't point to a map value\n", i);

kernel/bpf/verifier.c

13846

ret = process_timer_kfunc(env, regno, meta);

kernel/bpf/verifier.c

13852

verbose(env, "arg#%d doesn't point to a map value\n", i);

kernel/bpf/verifier.c

13855

ret = check_map_field_pointer(env, regno, BPF_TASK_WORK, &meta->map);

kernel/bpf/verifier.c

13861

verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i);

kernel/bpf/verifier.c

13864

ret = process_irq_flag(env, regno, meta);

kernel/bpf/verifier.c

13873

verbose(env, "arg#%d doesn't point to map value or allocated object\n", i);

kernel/bpf/verifier.c

13885

ret = process_spin_lock(env, regno, flags);

kernel/bpf/verifier.c

13894

verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",

kernel/bpf/verifier.c

13902

static int fetch_kfunc_arg_meta(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

13910

err = fetch_kfunc_meta(env, func_id, offset, &kfunc);

kernel/bpf/verifier.c

13920

if (!kfunc.flags || !btf_kfunc_is_allowed(kfunc.btf, kfunc.id, env->prog))

kernel/bpf/verifier.c

13933

static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,

kernel/bpf/verifier.c

13953

verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");

kernel/bpf/verifier.c

13957

ret_btf = env->prog->aux->btf;

kernel/bpf/verifier.c

13962

verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");

kernel/bpf/verifier.c

13968

verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");

kernel/bpf/verifier.c

13974

verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",

kernel/bpf/verifier.c

14003

if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {

kernel/bpf/verifier.c

14004

verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");

kernel/bpf/verifier.c

14009

verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");

kernel/bpf/verifier.c

14014

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14024

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14041

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14048

verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n",

kernel/bpf/verifier.c

14052

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14057

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14061

verbose(env,

kernel/bpf/verifier.c

14069

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14072

verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size");

kernel/bpf/verifier.c

14085

if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {

kernel/bpf/verifier.c

14086

verbose(env, "the prog does not allow writes to packet data\n");

kernel/bpf/verifier.c

14092

verifier_bug(env, "no dynptr id");

kernel/bpf/verifier.c

14108

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);

kernel/bpf/verifier.c

14110

static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

14115

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

14128

err = fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);

kernel/bpf/verifier.c

14130

verbose(env, "calling kernel function %s is not allowed\n", meta.func_name);

kernel/bpf/verifier.c

14135

insn_aux = &env->insn_aux_data[insn_idx];

kernel/bpf/verifier.c

14145

branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);

kernel/bpf/verifier.c

14147

verbose(env, "failed to push state for failed lock acquisition\n");

kernel/bpf/verifier.c

14155

mark_reg_not_init(env, regs, caller_saved[i]);

kernel/bpf/verifier.c

14157

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14158

err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1);

kernel/bpf/verifier.c

14160

verbose(env, "failed to mark s32 range for retval in forked state for lock\n");

kernel/bpf/verifier.c

14163

__mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32));

kernel/bpf/verifier.c

14165

verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n");

kernel/bpf/verifier.c

14170

verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");

kernel/bpf/verifier.c

14175

if (sleepable && !in_sleepable(env)) {

kernel/bpf/verifier.c

14176

verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);

kernel/bpf/verifier.c

14181

if (!in_sleepable_context(env))

kernel/bpf/verifier.c

14185

err = check_kfunc_args(env, &meta, insn_idx);

kernel/bpf/verifier.c

14190

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

14193

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

14205

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

14208

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

14215

err = push_callback_call(env, insn, insn_idx, meta.subprogno,

kernel/bpf/verifier.c

14218

verbose(env, "kfunc %s#%d failed callback verification\n",

kernel/bpf/verifier.c

14231

env->cur_state->active_rcu_locks++;

kernel/bpf/verifier.c

14237

if (env->cur_state->active_rcu_locks == 0) {

kernel/bpf/verifier.c

14238

verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);

kernel/bpf/verifier.c

14241

if (--env->cur_state->active_rcu_locks == 0) {

kernel/bpf/verifier.c

14242

bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({

kernel/bpf/verifier.c

14249

} else if (sleepable && env->cur_state->active_rcu_locks) {

kernel/bpf/verifier.c

14250

verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);

kernel/bpf/verifier.c

14254

if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {

kernel/bpf/verifier.c

14255

verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");

kernel/bpf/verifier.c

14259

if (env->cur_state->active_preempt_locks) {

kernel/bpf/verifier.c

14261

env->cur_state->active_preempt_locks++;

kernel/bpf/verifier.c

14263

env->cur_state->active_preempt_locks--;

kernel/bpf/verifier.c

14265

verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name);

kernel/bpf/verifier.c

14269

env->cur_state->active_preempt_locks++;

kernel/bpf/verifier.c

14271

verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);

kernel/bpf/verifier.c

14275

if (env->cur_state->active_irq_id && sleepable) {

kernel/bpf/verifier.c

14276

verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name);

kernel/bpf/verifier.c

14280

if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) {

kernel/bpf/verifier.c

14281

verbose(env, "kernel func %s requires RCU critical section protection\n", func_name);

kernel/bpf/verifier.c

14292

err = unmark_stack_slots_dynptr(env, reg);

kernel/bpf/verifier.c

14294

err = release_reference(env, reg->ref_obj_id);

kernel/bpf/verifier.c

14296

verbose(env, "kfunc %s#%d reference has not been acquired before\n",

kernel/bpf/verifier.c

14309

err = ref_convert_owning_non_owning(env, release_ref_obj_id);

kernel/bpf/verifier.c

14311

verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",

kernel/bpf/verifier.c

14316

err = release_reference(env, release_ref_obj_id);

kernel/bpf/verifier.c

14318

verbose(env, "kfunc %s#%d reference has not been acquired before\n",

kernel/bpf/verifier.c

14326

verbose(env, "JIT does not support calling kfunc %s#%d\n",

kernel/bpf/verifier.c

14330

env->seen_exception = true;

kernel/bpf/verifier.c

14335

if (!env->exception_callback_subprog) {

kernel/bpf/verifier.c

14336

err = check_return_code(env, BPF_REG_1, "R1");

kernel/bpf/verifier.c

14345

mark_reg_not_init(env, regs, regno);

kernel/bpf/verifier.c

14358

verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");

kernel/bpf/verifier.c

14364

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14367

__mark_reg_const_zero(env, &regs[BPF_REG_0]);

kernel/bpf/verifier.c

14368

mark_btf_func_reg_size(env, BPF_REG_0, t->size);

kernel/bpf/verifier.c

14371

err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);

kernel/bpf/verifier.c

14377

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14390

verbose(env,

kernel/bpf/verifier.c

14398

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14418

(get_iter_from_state(env->cur_state, &meta)

kernel/bpf/verifier.c

14440

mark_reg_known_zero(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

14449

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

14451

mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));

kernel/bpf/verifier.c

14453

int id = acquire_reference(env, insn_idx);

kernel/bpf/verifier.c

14461

ref_set_non_owning(env, &regs[BPF_REG_0]);

kernel/bpf/verifier.c

14465

regs[BPF_REG_0].id = ++env->id_gen;

kernel/bpf/verifier.c

14478

clear_all_pkt_pointers(env);

kernel/bpf/verifier.c

14487

mark_btf_func_reg_size(env, regno, sizeof(void *));

kernel/bpf/verifier.c

14490

mark_btf_func_reg_size(env, regno, t->size);

kernel/bpf/verifier.c

14494

err = process_iter_next_call(env, insn_idx, &meta);

kernel/bpf/verifier.c

14500

env->prog->call_session_cookie = true;

kernel/bpf/verifier.c

14505

static bool check_reg_sane_offset(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14514

verbose(env, "math between %s pointer and %lld is not allowed\n",

kernel/bpf/verifier.c

14515

reg_type_str(env, type), val);

kernel/bpf/verifier.c

14520

verbose(env, "%s pointer offset %d is not allowed\n",

kernel/bpf/verifier.c

14521

reg_type_str(env, type), reg->off);

kernel/bpf/verifier.c

14526

verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",

kernel/bpf/verifier.c

14527

reg_type_str(env, type));

kernel/bpf/verifier.c

14532

verbose(env, "value %lld makes %s pointer be out of bounds\n",

kernel/bpf/verifier.c

14533

smin, reg_type_str(env, type));

kernel/bpf/verifier.c

14579

static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14582

return env->bypass_spec_v1 ||

kernel/bpf/verifier.c

14584

cur_aux(env)->nospec;

kernel/bpf/verifier.c

14604

static int sanitize_val_alu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14607

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

14609

if (can_skip_alu_sanitation(env, insn))

kernel/bpf/verifier.c

14625

static int sanitize_speculative_path(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14632

branch = push_stack(env, next_idx, curr_idx, true);

kernel/bpf/verifier.c

14636

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

14638

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

14639

mark_reg_unknown(env, regs, insn->src_reg);

kernel/bpf/verifier.c

14645

static int sanitize_ptr_alu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14653

struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;

kernel/bpf/verifier.c

14654

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

14663

if (can_skip_alu_sanitation(env, insn))

kernel/bpf/verifier.c

14702

env->explore_alu_limits = true;

kernel/bpf/verifier.c

14733

err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);

kernel/bpf/verifier.c

14741

static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

14743

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

14751

env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;

kernel/bpf/verifier.c

14754

static int sanitize_err(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14765

verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",

kernel/bpf/verifier.c

14769

verbose(env, "R%d has pointer with unsupported alu operation, %s\n",

kernel/bpf/verifier.c

14773

verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",

kernel/bpf/verifier.c

14777

verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",

kernel/bpf/verifier.c

14781

verbose(env, "R%d could not be pushed for speculative verification, %s\n",

kernel/bpf/verifier.c

14785

verifier_bug(env, "unknown reason (%d)", reason);

kernel/bpf/verifier.c

14803

struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14812

verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",

kernel/bpf/verifier.c

14818

verbose(env, "R%d stack pointer arithmetic goes out of range, "

kernel/bpf/verifier.c

14826

static int sanitize_check_bounds(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14835

if (env->bypass_spec_v1)

kernel/bpf/verifier.c

14840

if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,

kernel/bpf/verifier.c

14845

if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {

kernel/bpf/verifier.c

14846

verbose(env, "R%d pointer arithmetic of map value goes out of range, "

kernel/bpf/verifier.c

14863

static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

14868

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

14888

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

14894

if (opcode == BPF_SUB && env->allow_ptr_leaks) {

kernel/bpf/verifier.c

14895

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

14899

verbose(env,

kernel/bpf/verifier.c

1490

static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)

kernel/bpf/verifier.c

14906

verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",

kernel/bpf/verifier.c

14907

dst, reg_type_str(env, ptr_reg->type));

kernel/bpf/verifier.c

14942

verbose(env, "R%d pointer arithmetic on %s prohibited\n",

kernel/bpf/verifier.c

14943

dst, reg_type_str(env, ptr_reg->type));

kernel/bpf/verifier.c

14953

if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||

kernel/bpf/verifier.c

14954

!check_reg_sane_offset(env, ptr_reg, ptr_reg->type))

kernel/bpf/verifier.c

14961

ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,

kernel/bpf/verifier.c

14964

return sanitize_err(env, insn, ret, off_reg, dst_reg);

kernel/bpf/verifier.c

15007

dst_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

15015

verbose(env, "R%d tried to subtract pointer from scalar\n",

kernel/bpf/verifier.c

15024

verbose(env, "R%d subtraction from stack pointer prohibited\n",

kernel/bpf/verifier.c

15063

dst_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

15073

verbose(env, "R%d bitwise operator %s on pointer prohibited\n",

kernel/bpf/verifier.c

15078

verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",

kernel/bpf/verifier.c

1508

if (env->subprog_info[state->subprogno].stack_depth < size)

kernel/bpf/verifier.c

15083

if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))

kernel/bpf/verifier.c

15086

bounds_ret = sanitize_check_bounds(env, insn, dst_reg);

kernel/bpf/verifier.c

1509

env->subprog_info[state->subprogno].stack_depth = size;

kernel/bpf/verifier.c

15090

ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,

kernel/bpf/verifier.c

15092

if (verifier_bug_if(!can_skip_alu_sanitation(env, insn)

kernel/bpf/verifier.c

15093

&& !env->cur_state->speculative

kernel/bpf/verifier.c

15096

env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) {

kernel/bpf/verifier.c

15100

return sanitize_err(env, insn, ret, off_reg, dst_reg);

kernel/bpf/verifier.c

1519

static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

1521

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

1533

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

1537

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

1541

s->id = ++env->id_gen;

kernel/bpf/verifier.c

1545

static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,

kernel/bpf/verifier.c

1548

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

1551

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

1564

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

1566

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

1569

s = acquire_reference_state(env, insn_idx);

kernel/bpf/verifier.c

1573

s->id = ++env->id_gen;

kernel/bpf/verifier.c

16010

static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

16024

branch = push_stack(env, env->insn_idx, env->insn_idx, false);

kernel/bpf/verifier.c

16043

static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

16054

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

16059

ret = sanitize_val_alu(env, insn);

kernel/bpf/verifier.c

16061

return sanitize_err(env, insn, ret, NULL, NULL);

kernel/bpf/verifier.c

16090

env->fake_reg[0] = *dst_reg;

kernel/bpf/verifier.c

16092

scalar32_min_max_sub(dst_reg, &env->fake_reg[0]);

kernel/bpf/verifier.c

16093

scalar_min_max_sub(dst_reg, &env->fake_reg[0]);

kernel/bpf/verifier.c

16094

dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off);

kernel/bpf/verifier.c

16135

ret = maybe_fork_scalars(env, insn, dst_reg);

kernel/bpf/verifier.c

16145

ret = maybe_fork_scalars(env, insn, dst_reg);

kernel/bpf/verifier.c

16201

static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

16204

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

16216

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

16240

if (opcode == BPF_SUB && env->allow_ptr_leaks) {

kernel/bpf/verifier.c

16241

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

16244

verbose(env, "R%d pointer %s pointer prohibited\n",

kernel/bpf/verifier.c

16253

err = mark_chain_precision(env, insn->dst_reg);

kernel/bpf/verifier.c

16256

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

16261

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

16264

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

16268

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

16280

return adjust_ptr_min_max_vals(env, insn,

kernel/bpf/verifier.c

16286

print_verifier_state(env, vstate, vstate->curframe, true);

kernel/bpf/verifier.c

16287

verbose(env, "verifier internal error: unexpected ptr_reg\n");

kernel/bpf/verifier.c

16291

print_verifier_state(env, vstate, vstate->curframe, true);

kernel/bpf/verifier.c

16292

verbose(env, "verifier internal error: no src_reg\n");

kernel/bpf/verifier.c

16302

err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);

kernel/bpf/verifier.c

16314

if (env->bpf_capable &&

kernel/bpf/verifier.c

16361

static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

16363

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

16372

verbose(env, "BPF_NEG uses reserved fields\n");

kernel/bpf/verifier.c

16380

verbose(env, "BPF_END uses reserved fields\n");

kernel/bpf/verifier.c

16386

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

16390

if (is_pointer_value(env, insn->dst_reg)) {

kernel/bpf/verifier.c

16391

verbose(env, "R%d pointer arithmetic prohibited\n",

kernel/bpf/verifier.c

16399

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

16400

err = err ?: adjust_scalar_min_max_vals(env, insn,

kernel/bpf/verifier.c

16404

err = check_reg_arg(env, insn->dst_reg, DST_OP);

kernel/bpf/verifier.c

16415

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

16420

verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");

kernel/bpf/verifier.c

16423

if (!env->prog->aux->arena) {

kernel/bpf/verifier.c

16424

verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");

kernel/bpf/verifier.c

16430

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

16436

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

16441

verbose(env, "BPF_MOV uses reserved fields\n");

kernel/bpf/verifier.c

16447

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

16458

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

16462

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

16468

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

16473

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

16474

verbose(env,

kernel/bpf/verifier.c

16483

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

16490

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

16495

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

16496

verbose(env,

kernel/bpf/verifier.c

16505

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

16513

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

16519

assign_scalar_id_before_mov(env, src_reg);

kernel/bpf/verifier.c

16523

dst_reg->subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

16527

mark_reg_unknown(env, regs,

kernel/bpf/verifier.c

16538

mark_reg_unknown(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

16550

verbose(env, "invalid BPF_ALU opcode %x\n", opcode);

kernel/bpf/verifier.c

16558

verbose(env, "BPF_ALU uses reserved fields\n");

kernel/bpf/verifier.c

16562

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

16568

verbose(env, "BPF_ALU uses reserved fields\n");

kernel/bpf/verifier.c

16574

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

16580

verbose(env, "div by zero\n");

kernel/bpf/verifier.c

16589

verbose(env, "invalid shift %d\n", insn->imm);

kernel/bpf/verifier.c

16595

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

16596

err = err ?: adjust_reg_min_max_vals(env, insn);

kernel/bpf/verifier.c

16601

return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");

kernel/bpf/verifier.c

1672

static void update_peak_states(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

1676

cur_states = env->explored_states_size + env->free_list_size + env->num_backedges;

kernel/bpf/verifier.c

1677

env->peak_states = max(env->peak_states, cur_states);

kernel/bpf/verifier.c

17154

static int reg_set_min_max(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17187

err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");

kernel/bpf/verifier.c

17188

err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");

kernel/bpf/verifier.c

17189

err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");

kernel/bpf/verifier.c

17190

err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");

kernel/bpf/verifier.c

1721

static bool incomplete_read_marks(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1728

static void maybe_free_verifier_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1733

|| incomplete_read_marks(env, &sl->state))

kernel/bpf/verifier.c

1738

env->free_list_size--;

kernel/bpf/verifier.c

17391

static void collect_linked_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17396

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

17424

static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate,

kernel/bpf/verifier.c

17478

mark_reg_scratched(env, e->regno);

kernel/bpf/verifier.c

17480

mark_stack_slot_scratched(env, e->spi);

kernel/bpf/verifier.c

17484

static int check_cond_jmp_op(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

17487

struct bpf_verifier_state *this_branch = env->cur_state;

kernel/bpf/verifier.c

17501

verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);

kernel/bpf/verifier.c

17506

struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;

kernel/bpf/verifier.c

17512

verbose(env, "invalid may_goto imm %d\n", insn->imm);

kernel/bpf/verifier.c

17515

prev_st = find_prev_entry(env, cur_st->parent, idx);

kernel/bpf/verifier.c

17518

queued_st = push_stack(env, idx + 1, idx, false);

kernel/bpf/verifier.c

17524

widen_imprecise_scalars(env, prev_st, queued_st);

kernel/bpf/verifier.c

17530

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

17537

verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");

kernel/bpf/verifier.c

17542

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

17548

is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

17549

verbose(env, "R%d pointer comparison prohibited\n",

kernel/bpf/verifier.c

17560

verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");

kernel/bpf/verifier.c

17563

src_reg = &env->fake_reg[0];

kernel/bpf/verifier.c

17573

err = push_jmp_history(env, this_branch, insn_flags, 0);

kernel/bpf/verifier.c

17585

err = mark_chain_precision(env, insn->dst_reg);

kernel/bpf/verifier.c

17588

err = mark_chain_precision(env, insn->src_reg);

kernel/bpf/verifier.c

17598

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

17599

err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx);

kernel/bpf/verifier.c

17603

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

17604

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

17612

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

17613

err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1,

kernel/bpf/verifier.c

17618

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

17619

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

17629

collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);

kernel/bpf/verifier.c

17631

collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);

kernel/bpf/verifier.c

17633

err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));

kernel/bpf/verifier.c

17638

other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);

kernel/bpf/verifier.c

17644

err = reg_set_min_max(env,

kernel/bpf/verifier.c

17653

memcpy(&env->fake_reg[1], &env->fake_reg[0],

kernel/bpf/verifier.c

17654

sizeof(env->fake_reg[0]));

kernel/bpf/verifier.c

17655

err = reg_set_min_max(env,

kernel/bpf/verifier.c

17657

&env->fake_reg[0],

kernel/bpf/verifier.c

17658

dst_reg, &env->fake_reg[1],

kernel/bpf/verifier.c

17667

sync_linked_regs(env, this_branch, src_reg, &linked_regs);

kernel/bpf/verifier.c

17668

sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg],

kernel/bpf/verifier.c

17673

sync_linked_regs(env, this_branch, dst_reg, &linked_regs);

kernel/bpf/verifier.c

17674

sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg],

kernel/bpf/verifier.c

17731

is_pointer_value(env, insn->dst_reg)) {

kernel/bpf/verifier.c

17732

verbose(env, "R%d pointer comparison prohibited\n",

kernel/bpf/verifier.c

17736

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

17737

print_insn_state(env, this_branch, this_branch->curframe);

kernel/bpf/verifier.c

17742

static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

17744

struct bpf_insn_aux_data *aux = cur_aux(env);

kernel/bpf/verifier.c

17745

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

17751

verbose(env, "invalid BPF_LD_IMM insn\n");

kernel/bpf/verifier.c

17755

verbose(env, "BPF_LD_IMM64 uses reserved fields\n");

kernel/bpf/verifier.c

17759

err = check_reg_arg(env, insn->dst_reg, DST_OP);

kernel/bpf/verifier.c

17776

mark_reg_known_zero(env, regs, insn->dst_reg);

kernel/bpf/verifier.c

17789

verifier_bug(env, "pseudo btf id: unexpected dst reg type");

kernel/bpf/verifier.c

17796

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

17797

u32 subprogno = find_subprog(env,

kernel/bpf/verifier.c

17798

env->insn_idx + insn->imm + 1);

kernel/bpf/verifier.c

17801

verbose(env, "missing btf func_info\n");

kernel/bpf/verifier.c

17805

verbose(env, "callback function not static\n");

kernel/bpf/verifier.c

17814

map = env->used_maps[aux->map_index];

kernel/bpf/verifier.c

17820

__mark_reg_unknown(env, dst_reg);

kernel/bpf/verifier.c

17832

verifier_bug(env, "unexpected src reg value for ldimm64");

kernel/bpf/verifier.c

17866

static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

17868

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

17873

if (!may_access_skb(resolve_prog_type(env->prog))) {

kernel/bpf/verifier.c

17874

verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");

kernel/bpf/verifier.c

17878

if (!env->ops->gen_ld_abs) {

kernel/bpf/verifier.c

17879

verifier_bug(env, "gen_ld_abs is null");

kernel/bpf/verifier.c

17886

verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");

kernel/bpf/verifier.c

17891

err = check_reg_arg(env, ctx_reg, SRC_OP);

kernel/bpf/verifier.c

17899

err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");

kernel/bpf/verifier.c

17904

verbose(env,

kernel/bpf/verifier.c

17911

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

17916

err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);

kernel/bpf/verifier.c

17922

mark_reg_not_init(env, regs, caller_saved[i]);

kernel/bpf/verifier.c

17923

check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);

kernel/bpf/verifier.c

17930

mark_reg_unknown(env, regs, BPF_REG_0);

kernel/bpf/verifier.c

17932

regs[BPF_REG_0].subreg_def = env->insn_idx + 1;

kernel/bpf/verifier.c

17936

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)

kernel/bpf/verifier.c

17940

const struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

17941

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

17943

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

17945

struct bpf_func_state *frame = env->cur_state->frame[0];

kernel/bpf/verifier.c

17976

return __check_ptr_off_reg(env, reg, regno, false);

kernel/bpf/verifier.c

17989

err = check_reg_arg(env, regno, SRC_OP);

kernel/bpf/verifier.c

17993

if (is_pointer_value(env, regno)) {

kernel/bpf/verifier.c

17994

verbose(env, "R%d leaks addr as return value\n", regno);

kernel/bpf/verifier.c

18006

verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",

kernel/bpf/verifier.c

18007

regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

1801

static u32 state_htab_size(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18015

if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||

kernel/bpf/verifier.c

18016

env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||

kernel/bpf/verifier.c

18017

env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||

kernel/bpf/verifier.c

18018

env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||

kernel/bpf/verifier.c

18019

env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||

kernel/bpf/verifier.c

18020

env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||

kernel/bpf/verifier.c

18021

env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||

kernel/bpf/verifier.c

18022

env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||

kernel/bpf/verifier.c

18023

env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)

kernel/bpf/verifier.c

18025

if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||

kernel/bpf/verifier.c

18026

env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)

kernel/bpf/verifier.c

1803

return env->prog->len;

kernel/bpf/verifier.c

18030

if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {

kernel/bpf/verifier.c

18042

if (!env->prog->aux->attach_btf_id)

kernel/bpf/verifier.c

18047

switch (env->prog->expected_attach_type) {

kernel/bpf/verifier.c

1806

static struct list_head *explored_state(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

18063

switch (env->prog->expected_attach_type) {

kernel/bpf/verifier.c

18077

if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {

kernel/bpf/verifier.c

18079

if (!get_func_retval_range(env->prog, &range))

kernel/bpf/verifier.c

1808

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

18085

} else if (!env->prog->aux->attach_func_proto->type) {

kernel/bpf/verifier.c

1811

return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];

kernel/bpf/verifier.c

18111

verbose(env, "%s the register R%d is not a known value (%s)\n",

kernel/bpf/verifier.c

18112

exit_ctx, regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

18116

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

18121

verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);

kernel/bpf/verifier.c

18126

verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");

kernel/bpf/verifier.c

18132

env->prog->enforce_expected_attach_type = 1;

kernel/bpf/verifier.c

18136

static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

18140

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

18144

static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

18148

subprog = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

18158

static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)

kernel/bpf/verifier.c

18162

caller = bpf_find_containing_subprog(env, t);

kernel/bpf/verifier.c

18163

callee = bpf_find_containing_subprog(env, w);

kernel/bpf/verifier.c

18208

static void mark_prune_point(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

18210

env->insn_aux_data[idx].prune_point = true;

kernel/bpf/verifier.c

18213

static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

18215

return env->insn_aux_data[insn_idx].prune_point;

kernel/bpf/verifier.c

18218

static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

18220

env->insn_aux_data[idx].force_checkpoint = true;

kernel/bpf/verifier.c

18223

static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

18225

return env->insn_aux_data[insn_idx].force_checkpoint;

kernel/bpf/verifier.c

18228

static void mark_calls_callback(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

18230

env->insn_aux_data[idx].calls_callback = true;

kernel/bpf/verifier.c

18233

bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

18235

return env->insn_aux_data[insn_idx].calls_callback;

kernel/bpf/verifier.c

18248

static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18250

int *insn_stack = env->cfg.insn_stack;

kernel/bpf/verifier.c

18251

int *insn_state = env->cfg.insn_state;

kernel/bpf/verifier.c

18259

if (w < 0 || w >= env->prog->len) {

kernel/bpf/verifier.c

18260

verbose_linfo(env, t, "%d: ", t);

kernel/bpf/verifier.c

18261

verbose(env, "jump out of range from insn %d to %d\n", t, w);

kernel/bpf/verifier.c

18267

mark_prune_point(env, w);

kernel/bpf/verifier.c

18268

mark_jmp_point(env, w);

kernel/bpf/verifier.c

18275

if (env->cfg.cur_stack >= env->prog->len)

kernel/bpf/verifier.c

18277

insn_stack[env->cfg.cur_stack++] = w;

kernel/bpf/verifier.c

18280

if (env->bpf_capable)

kernel/bpf/verifier.c

18282

verbose_linfo(env, t, "%d: ", t);

kernel/bpf/verifier.c

18283

verbose_linfo(env, w, "%d: ", w);

kernel/bpf/verifier.c

18284

verbose(env, "back-edge from insn %d to %d\n", t, w);

kernel/bpf/verifier.c

18290

verifier_bug(env, "insn state internal bug");

kernel/bpf/verifier.c

18297

struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18304

ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);

kernel/bpf/verifier.c

18308

mark_prune_point(env, t + insn_sz);

kernel/bpf/verifier.c

18310

mark_jmp_point(env, t + insn_sz);

kernel/bpf/verifier.c

18314

mark_prune_point(env, t);

kernel/bpf/verifier.c

18315

merge_callee_effects(env, t, w);

kernel/bpf/verifier.c

18316

ret = push_insn(t, w, BRANCH, env);

kernel/bpf/verifier.c

18328

static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)

kernel/bpf/verifier.c

18337

return env->prog->jit_requested && bpf_jit_supports_percpu_insn();

kernel/bpf/verifier.c

18353

static bool get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,

kernel/bpf/verifier.c

18362

if (get_helper_proto(env, call->imm, &fn) < 0)

kernel/bpf/verifier.c

18366

(verifier_inlines_helper_call(env, call->imm) ||

kernel/bpf/verifier.c

18381

err = fetch_kfunc_arg_meta(env, call->imm, call->off, &meta);

kernel/bpf/verifier.c

18468

static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18472

struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;

kernel/bpf/verifier.c

18473

struct bpf_insn *call = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

18480

if (!get_call_summary(env, call, &cs))

kernel/bpf/verifier.c

1849

static bool compute_scc_callchain(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18503

if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)

kernel/bpf/verifier.c

18526

env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;

kernel/bpf/verifier.c

18527

env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;

kernel/bpf/verifier.c

18549

env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;

kernel/bpf/verifier.c

18555

static int mark_fastcall_patterns(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18557

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

18562

for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {

kernel/bpf/verifier.c

18566

insn = env->prog->insnsi + i;

kernel/bpf/verifier.c

18574

insn = env->prog->insnsi + i;

kernel/bpf/verifier.c

18577

mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);

kernel/bpf/verifier.c

1858

scc = env->insn_aux_data[insn_idx].scc;

kernel/bpf/verifier.c

18687

static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18695

for (i = 0; i < env->insn_array_map_cnt; i++) {

kernel/bpf/verifier.c

18700

map = env->insn_array_maps[i];

kernel/bpf/verifier.c

1872

static struct bpf_scc_visit *scc_visit_lookup(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18726

verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);

kernel/bpf/verifier.c

18735

create_jt(int t, struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18742

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/verifier.c

18745

jt = jt_from_subprog(env, subprog_start, subprog_end);

kernel/bpf/verifier.c

1875

struct bpf_scc_info *info = env->scc_info[callchain->scc];

kernel/bpf/verifier.c

18752

verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",

kernel/bpf/verifier.c

18763

static int visit_gotox_insn(int t, struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18765

int *insn_stack = env->cfg.insn_stack;

kernel/bpf/verifier.c

18766

int *insn_state = env->cfg.insn_state;

kernel/bpf/verifier.c

18771

jt = env->insn_aux_data[t].jt;

kernel/bpf/verifier.c

18773

jt = create_jt(t, env);

kernel/bpf/verifier.c

18777

env->insn_aux_data[t].jt = jt;

kernel/bpf/verifier.c

18780

mark_prune_point(env, t);

kernel/bpf/verifier.c

18783

if (w < 0 || w >= env->prog->len) {

kernel/bpf/verifier.c

18784

verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);

kernel/bpf/verifier.c

18788

mark_jmp_point(env, w);

kernel/bpf/verifier.c

18794

if (env->cfg.cur_stack >= env->prog->len)

kernel/bpf/verifier.c

18797

insn_stack[env->cfg.cur_stack++] = w;

kernel/bpf/verifier.c

18805

static int visit_tailcall_insn(struct bpf_verifier_env *env, int t)

kernel/bpf/verifier.c

18810

if (env->insn_aux_data[t].jt)

kernel/bpf/verifier.c

18817

subprog = bpf_find_containing_subprog(env, t);

kernel/bpf/verifier.c

18820

env->insn_aux_data[t].jt = jt;

kernel/bpf/verifier.c

18829

static int visit_insn(int t, struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18831

struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];

kernel/bpf/verifier.c

18835

return visit_func_call_insn(t, insns, env, true);

kernel/bpf/verifier.c

18841

return push_insn(t, t + insn_sz, FALLTHROUGH, env);

kernel/bpf/verifier.c

18855

mark_prune_point(env, t);

kernel/bpf/verifier.c

18866

mark_calls_callback(env, t);

kernel/bpf/verifier.c

18867

mark_force_checkpoint(env, t);

kernel/bpf/verifier.c

18868

mark_prune_point(env, t);

kernel/bpf/verifier.c

18869

mark_jmp_point(env, t);

kernel/bpf/verifier.c

18874

ret = get_helper_proto(env, insn->imm, &fp);

kernel/bpf/verifier.c

18881

mark_subprog_might_sleep(env, t);

kernel/bpf/verifier.c

18883

mark_subprog_changes_pkt_data(env, t);

kernel/bpf/verifier.c

18885

visit_tailcall_insn(env, t);

kernel/bpf/verifier.c

18889

ret = fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);

kernel/bpf/verifier.c

18891

mark_prune_point(env, t);

kernel/bpf/verifier.c

18903

mark_force_checkpoint(env, t);

kernel/bpf/verifier.c

1891

static struct bpf_scc_visit *scc_visit_alloc(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

18911

mark_subprog_might_sleep(env, t);

kernel/bpf/verifier.c

18913

mark_subprog_changes_pkt_data(env, t);

kernel/bpf/verifier.c

18915

return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);

kernel/bpf/verifier.c

18919

return visit_gotox_insn(t, env);

kernel/bpf/verifier.c

18927

ret = push_insn(t, t + off + 1, FALLTHROUGH, env);

kernel/bpf/verifier.c

18931

mark_prune_point(env, t + off + 1);

kernel/bpf/verifier.c

18932

mark_jmp_point(env, t + off + 1);

kernel/bpf/verifier.c

18938

mark_prune_point(env, t);

kernel/bpf/verifier.c

18940

mark_force_checkpoint(env, t);

kernel/bpf/verifier.c

18942

ret = push_insn(t, t + 1, FALLTHROUGH, env);

kernel/bpf/verifier.c

18946

return push_insn(t, t + insn->off + 1, BRANCH, env);

kernel/bpf/verifier.c

18953

static int check_cfg(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

18955

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

18959

insn_state = env->cfg.insn_state = kvzalloc_objs(int, insn_cnt,

kernel/bpf/verifier.c

18964

insn_stack = env->cfg.insn_stack = kvzalloc_objs(int, insn_cnt,

kernel/bpf/verifier.c

18971

ex_insn_beg = env->exception_callback_subprog

kernel/bpf/verifier.c

18972

? env->subprog_info[env->exception_callback_subprog].start

kernel/bpf/verifier.c

18977

env->cfg.cur_stack = 1;

kernel/bpf/verifier.c

18980

while (env->cfg.cur_stack > 0) {

kernel/bpf/verifier.c

18981

int t = insn_stack[env->cfg.cur_stack - 1];

kernel/bpf/verifier.c

18983

ret = visit_insn(t, env);

kernel/bpf/verifier.c

18987

env->cfg.cur_stack--;

kernel/bpf/verifier.c

18993

verifier_bug(env, "visit_insn internal bug");

kernel/bpf/verifier.c

1900

info = env->scc_info[scc];

kernel/bpf/verifier.c

19000

if (env->cfg.cur_stack < 0) {

kernel/bpf/verifier.c

19001

verifier_bug(env, "pop stack internal bug");

kernel/bpf/verifier.c

19009

env->cfg.cur_stack = 1;

kernel/bpf/verifier.c

19014

struct bpf_insn *insn = &env->prog->insnsi[i];

kernel/bpf/verifier.c

19017

verbose(env, "unreachable insn %d\n", i);

kernel/bpf/verifier.c

19023

verbose(env, "jump into the middle of ldimm64 insn %d\n", i);

kernel/bpf/verifier.c

1903

info = kvrealloc(env->scc_info[scc], new_sz, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

19031

env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;

kernel/bpf/verifier.c

19032

env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;

kernel/bpf/verifier.c

19037

env->cfg.insn_state = env->cfg.insn_stack = NULL;

kernel/bpf/verifier.c

19046

static int compute_postorder(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

19052

postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

19053

state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

19054

stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

1906

env->scc_info[scc] = info;

kernel/bpf/verifier.c

19062

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

19063

env->subprog_info[i].postorder_start = cur_postorder;

kernel/bpf/verifier.c

19064

stack[0] = env->subprog_info[i].start;

kernel/bpf/verifier.c

19074

succ = bpf_insn_successors(env, top);

kernel/bpf/verifier.c

19084

env->subprog_info[i].postorder_start = cur_postorder;

kernel/bpf/verifier.c

19085

env->cfg.insn_postorder = postorder;

kernel/bpf/verifier.c

19086

env->cfg.cur_postorder = cur_postorder;

kernel/bpf/verifier.c

19092

static int check_abnormal_return(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

19096

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

19097

if (env->subprog_info[i].has_ld_abs) {

kernel/bpf/verifier.c

19098

verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");

kernel/bpf/verifier.c

19101

if (env->subprog_info[i].has_tail_call) {

kernel/bpf/verifier.c

19102

verbose(env, "tail_call is not allowed in subprogs without BTF\n");

kernel/bpf/verifier.c

19113

static int check_btf_func_early(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19129

if (check_abnormal_return(env))

kernel/bpf/verifier.c

19138

verbose(env, "invalid func info rec size %u\n", urec_size);

kernel/bpf/verifier.c

19142

prog = env->prog;

kernel/bpf/verifier.c

1915

static char *format_callchain(struct bpf_verifier_env *env, struct bpf_scc_callchain *callchain)

kernel/bpf/verifier.c

19156

verbose(env, "nonzero tailing record in func info");

kernel/bpf/verifier.c

1917

char *buf = env->tmp_str_buf;

kernel/bpf/verifier.c

19177

verbose(env,

kernel/bpf/verifier.c

19183

verbose(env,

kernel/bpf/verifier.c

19192

verbose(env, "invalid type id %d in func info",

kernel/bpf/verifier.c

19215

static int check_btf_func(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19231

if (check_abnormal_return(env))

kernel/bpf/verifier.c

19235

if (nfuncs != env->subprog_cnt) {

kernel/bpf/verifier.c

19236

verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");

kernel/bpf/verifier.c

19242

prog = env->prog;

kernel/bpf/verifier.c

19257

if (env->subprog_info[i].start != krecord[i].insn_off) {

kernel/bpf/verifier.c

19258

verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");

kernel/bpf/verifier.c

19271

if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {

kernel/bpf/verifier.c

19272

verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");

kernel/bpf/verifier.c

19275

if (i && !scalar_return && env->subprog_info[i].has_tail_call) {

kernel/bpf/verifier.c

19276

verbose(env, "tail_call is only allowed in functions that return 'int'.\n");

kernel/bpf/verifier.c

1928

return env->tmp_str_buf;

kernel/bpf/verifier.c

19291

static void adjust_btf_func(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

19293

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

19300

for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)

kernel/bpf/verifier.c

19301

aux->func_info[i].insn_off = env->subprog_info[i].start;

kernel/bpf/verifier.c

19307

static int check_btf_line(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19339

prog = env->prog;

kernel/bpf/verifier.c

19343

sub = env->subprog_info;

kernel/bpf/verifier.c

1935

static int maybe_enter_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/verifier.c

19351

verbose(env, "nonzero tailing record in line_info");

kernel/bpf/verifier.c

1937

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/verifier.c

19378

verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",

kernel/bpf/verifier.c

19386

verbose(env,

kernel/bpf/verifier.c

19395

verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);

kernel/bpf/verifier.c

1940

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/verifier.c

19400

if (s != env->subprog_cnt) {

kernel/bpf/verifier.c

19405

verbose(env, "missing bpf_line_info for func#%u\n", s);

kernel/bpf/verifier.c

19415

if (s != env->subprog_cnt) {

kernel/bpf/verifier.c

19416

verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",

kernel/bpf/verifier.c

19417

env->subprog_cnt - s, s);

kernel/bpf/verifier.c

1942

visit = scc_visit_lookup(env, callchain);

kernel/bpf/verifier.c

1943

visit = visit ?: scc_visit_alloc(env, callchain);

kernel/bpf/verifier.c

19435

static int check_core_relo(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19441

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

19444

.log = &env->log,

kernel/bpf/verifier.c

19474

verbose(env, "nonzero tailing record in core_relo");

kernel/bpf/verifier.c

1948

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

19489

verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",

kernel/bpf/verifier.c

1949

verbose(env, "SCC enter %s\n", format_callchain(env, callchain));

kernel/bpf/verifier.c

19504

static int check_btf_info_early(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19512

if (check_abnormal_return(env))

kernel/bpf/verifier.c

19524

env->prog->aux->btf = btf;

kernel/bpf/verifier.c

19526

err = check_btf_func_early(env, attr, uattr);

kernel/bpf/verifier.c

19532

static int check_btf_info(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19539

if (check_abnormal_return(env))

kernel/bpf/verifier.c

1954

static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit);

kernel/bpf/verifier.c

19544

err = check_btf_func(env, attr, uattr);

kernel/bpf/verifier.c

19548

err = check_btf_line(env, attr, uattr);

kernel/bpf/verifier.c

19552

err = check_core_relo(env, attr, uattr);

kernel/bpf/verifier.c

1960

static int maybe_exit_scc(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/verifier.c

1962

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/verifier.c

19643

static void clean_func_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19647

u16 live_regs = env->insn_aux_data[ip].live_regs_before;

kernel/bpf/verifier.c

1965

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/verifier.c

19656

__mark_reg_not_init(env, &st->regs[i]);

kernel/bpf/verifier.c

19660

if (!bpf_stack_slot_alive(env, st->frameno, i)) {

kernel/bpf/verifier.c

19661

__mark_reg_not_init(env, &st->stack[i].spilled_ptr);

kernel/bpf/verifier.c

19668

static void clean_verifier_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

1967

visit = scc_visit_lookup(env, callchain);

kernel/bpf/verifier.c

19673

bpf_live_stack_query_init(env, st);

kernel/bpf/verifier.c

19677

clean_func_state(env, st->frame[i], ip);

kernel/bpf/verifier.c

19748

static void clear_singular_ids(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19751

struct bpf_idset *idset = &env->idset_scratch;

kernel/bpf/verifier.c

19777

static void clean_live_states(struct bpf_verifier_env *env, int insn,

kernel/bpf/verifier.c

19783

head = explored_state(env, insn);

kernel/bpf/verifier.c

19794

if (incomplete_read_marks(env, &sl->state))

kernel/bpf/verifier.c

19796

clean_verifier_state(env, &sl->state);

kernel/bpf/verifier.c

19816

static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,

kernel/bpf/verifier.c

1982

verifier_bug(env, "scc exit: no visit info for call chain %s",

kernel/bpf/verifier.c

1983

format_callchain(env, callchain));

kernel/bpf/verifier.c

19853

if (env->explore_alu_limits) {

kernel/bpf/verifier.c

1990

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

1991

verbose(env, "SCC exit %s\n", format_callchain(env, callchain));

kernel/bpf/verifier.c

1993

env->num_backedges -= visit->num_backedges;

kernel/bpf/verifier.c

1995

update_peak_states(env);

kernel/bpf/verifier.c

1996

return propagate_backedges(env, visit);

kernel/bpf/verifier.c

19983

static bool is_stack_all_misc(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

19990

(stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack))

kernel/bpf/verifier.c

19998

static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20004

if (is_stack_all_misc(env, stack))

kernel/bpf/verifier.c

20010

static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,

kernel/bpf/verifier.c

2002

static int add_scc_backedge(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20034

if (env->allow_uninit_stack &&

kernel/bpf/verifier.c

20049

old_reg = scalar_reg_for_stack(env, &old->stack[spi]);

kernel/bpf/verifier.c

20050

cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]);

kernel/bpf/verifier.c

20052

if (!regsafe(env, old_reg, cur_reg, idmap, exact))

kernel/bpf/verifier.c

2006

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/verifier.c

20088

if (!regsafe(env, &old->stack[spi].spilled_ptr,

kernel/bpf/verifier.c

2009

if (!compute_scc_callchain(env, st, callchain)) {

kernel/bpf/verifier.c

2010

verifier_bug(env, "add backedge: no SCC in verification path, insn_idx %d",

kernel/bpf/verifier.c

2014

visit = scc_visit_lookup(env, callchain);

kernel/bpf/verifier.c

2016

verifier_bug(env, "add backedge: no visit info for call chain %s",

kernel/bpf/verifier.c

2017

format_callchain(env, callchain));

kernel/bpf/verifier.c

2020

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

20208

static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,

kernel/bpf/verifier.c

2021

verbose(env, "SCC backedge %s\n", format_callchain(env, callchain));

kernel/bpf/verifier.c

20211

u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;

kernel/bpf/verifier.c

20219

!regsafe(env, &old->regs[i], &cur->regs[i],

kernel/bpf/verifier.c

20220

&env->idmap_scratch, exact))

kernel/bpf/verifier.c

20223

if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))

kernel/bpf/verifier.c

20229

static void reset_idmap_scratch(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

20231

struct bpf_idmap *idmap = &env->idmap_scratch;

kernel/bpf/verifier.c

20233

idmap->tmp_id_gen = env->id_gen;

kernel/bpf/verifier.c

20237

static bool states_equal(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20248

reset_idmap_scratch(env);

kernel/bpf/verifier.c

2025

env->num_backedges++;

kernel/bpf/verifier.c

20259

if (!refsafe(old, cur, &env->idmap_scratch))

kernel/bpf/verifier.c

2026

update_peak_states(env);

kernel/bpf/verifier.c

20269

if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))

kernel/bpf/verifier.c

20278

static int propagate_precision(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20296

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

20298

verbose(env, "frame %d: propagating r%d", fr, i);

kernel/bpf/verifier.c

20300

verbose(env, ",r%d", i);

kernel/bpf/verifier.c

20302

bt_set_frame_reg(&env->bt, fr, i);

kernel/bpf/verifier.c

20313

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

20315

verbose(env, "frame %d: propagating fp%d",

kernel/bpf/verifier.c

20318

verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);

kernel/bpf/verifier.c

20320

bt_set_frame_slot(&env->bt, fr, i);

kernel/bpf/verifier.c

20323

if (!first && (env->log.level & BPF_LOG_LEVEL2))

kernel/bpf/verifier.c

20324

verbose(env, "\n");

kernel/bpf/verifier.c

20327

err = __mark_chain_precision(env, cur, -1, changed);

kernel/bpf/verifier.c

2034

static bool incomplete_read_marks(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20342

static int propagate_backedges(struct bpf_verifier_env *env, struct bpf_scc_visit *visit)

kernel/bpf/verifier.c

20352

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

20353

verbose(env, "%s: too many iterations\n", __func__);

kernel/bpf/verifier.c

20355

mark_all_scalars_precise(env, &backedge->state);

kernel/bpf/verifier.c

20361

err = propagate_precision(env, st->equal_state, st, &changed);

kernel/bpf/verifier.c

2037

struct bpf_scc_callchain *callchain = &env->callchain_buf;

kernel/bpf/verifier.c

20389

static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

20391

return env->insn_aux_data[insn_idx].is_iter_next;

kernel/bpf/verifier.c

2040

if (!compute_scc_callchain(env, st, callchain))

kernel/bpf/verifier.c

2042

visit = scc_visit_lookup(env, callchain);

kernel/bpf/verifier.c

20476

static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

20480

struct bpf_verifier_state *cur = env->cur_state, *new;

kernel/bpf/verifier.c

20485

force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||

kernel/bpf/verifier.c

20498

if (env->jmps_processed - env->prev_jmps_processed >= 2 &&

kernel/bpf/verifier.c

20499

env->insn_processed - env->prev_insn_processed >= 8)

kernel/bpf/verifier.c

205

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);

kernel/bpf/verifier.c

20502

clean_live_states(env, insn_idx, cur);

kernel/bpf/verifier.c

20505

head = explored_state(env, insn_idx);

kernel/bpf/verifier.c

20566

if (is_iter_next_insn(env, insn_idx)) {

kernel/bpf/verifier.c

20567

if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/verifier.c

20582

iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;

kernel/bpf/verifier.c

20590

if (is_may_goto_insn_at(env, insn_idx)) {

kernel/bpf/verifier.c

20592

states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/verifier.c

20597

if (bpf_calls_callback(env, insn_idx)) {

kernel/bpf/verifier.c

20598

if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {

kernel/bpf/verifier.c

2060

static int update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/verifier.c

20606

states_equal(env, &sl->state, cur, EXACT) &&

kernel/bpf/verifier.c

20610

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/verifier.c

20611

verbose(env, "infinite loop detected at insn %d\n", insn_idx);

kernel/bpf/verifier.c

20612

verbose(env, "cur state:");

kernel/bpf/verifier.c

20613

print_verifier_state(env, cur, cur->curframe, true);

kernel/bpf/verifier.c

20614

verbose(env, "old state:");

kernel/bpf/verifier.c

20615

print_verifier_state(env, &sl->state, cur->curframe, true);

kernel/bpf/verifier.c

20632

env->jmps_processed - env->prev_jmps_processed < 20 &&

kernel/bpf/verifier.c

20633

env->insn_processed - env->prev_insn_processed < 100)

kernel/bpf/verifier.c

20638

loop = incomplete_read_marks(env, &sl->state);

kernel/bpf/verifier.c

20639

if (states_equal(env, &sl->state, cur, loop ? RANGE_WITHIN : NOT_EXACT)) {

kernel/bpf/verifier.c

20649

if (is_jmp_point(env, env->insn_idx))

kernel/bpf/verifier.c

20650

err = push_jmp_history(env, cur, 0, 0);

kernel/bpf/verifier.c

20651

err = err ? : propagate_precision(env, &sl->state, cur, NULL);

kernel/bpf/verifier.c

207

static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);

kernel/bpf/verifier.c

2072

verifier_bug_if((int)br < 0, env, "%s:branches_to_explore=%d", __func__, br);

kernel/bpf/verifier.c

20736

err = err ?: add_scc_backedge(env, &sl->state, backedge);

kernel/bpf/verifier.c

2075

err = maybe_exit_scc(env, st);

kernel/bpf/verifier.c

20762

n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;

kernel/bpf/verifier.c

20769

list_add(&sl->node, &env->free_list);

kernel/bpf/verifier.c

20770

env->free_list_size++;

kernel/bpf/verifier.c

20771

env->explored_states_size--;

kernel/bpf/verifier.c

20772

maybe_free_verifier_state(env, sl);

kernel/bpf/verifier.c

20776

if (env->max_states_per_insn < states_cnt)

kernel/bpf/verifier.c

20777

env->max_states_per_insn = states_cnt;

kernel/bpf/verifier.c

20779

if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)

kernel/bpf/verifier.c

20797

env->total_states++;

kernel/bpf/verifier.c

20798

env->explored_states_size++;

kernel/bpf/verifier.c

20799

update_peak_states(env);

kernel/bpf/verifier.c

208

static void invalidate_non_owning_refs(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

20800

env->prev_jmps_processed = env->jmps_processed;

kernel/bpf/verifier.c

20801

env->prev_insn_processed = env->insn_processed;

kernel/bpf/verifier.c

20804

if (env->bpf_capable)

kernel/bpf/verifier.c

20805

mark_all_scalars_imprecise(env, cur);

kernel/bpf/verifier.c

20807

clear_singular_ids(env, cur);

kernel/bpf/verifier.c

2081

maybe_free_verifier_state(env, sl);

kernel/bpf/verifier.c

20818

verifier_bug_if(new->branches != 1, env,

kernel/bpf/verifier.c

20821

err = maybe_enter_scc(env, new);

kernel/bpf/verifier.c

2088

static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,

kernel/bpf/verifier.c

20887

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,

kernel/bpf/verifier.c

20890

enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;

kernel/bpf/verifier.c

209

static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

2091

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

2092

struct bpf_verifier_stack_elem *elem, *head = env->head;

kernel/bpf/verifier.c

20927

verbose(env, "same insn cannot be used with different pointers\n");

kernel/bpf/verifier.c

20939

static int process_bpf_exit_full(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20949

int err = check_resource_leak(env, exception_exit,

kernel/bpf/verifier.c

2095

if (env->head == NULL)

kernel/bpf/verifier.c

20950

exception_exit || !env->cur_state->curframe,

kernel/bpf/verifier.c

20968

if (env->cur_state->curframe) {

kernel/bpf/verifier.c

20970

err = prepare_func_exit(env, &env->insn_idx);

kernel/bpf/verifier.c

20977

err = check_return_code(env, BPF_REG_0, "R0");

kernel/bpf/verifier.c

20983

static int indirect_jump_min_max_index(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

20988

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

20994

verbose(env, "the sum of R%u umin_value %llu and off %u is too big\n",

kernel/bpf/verifier.c

210

static int ref_set_non_owning(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21000

verbose(env, "the sum of R%u umax_value %llu and off %u is too big\n",

kernel/bpf/verifier.c

21009

verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n",

kernel/bpf/verifier.c

21020

static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

21030

dst_reg = reg_state(env, insn->dst_reg);

kernel/bpf/verifier.c

21032

verbose(env, "R%d has type %s, expected PTR_TO_INSN\n",

kernel/bpf/verifier.c

21033

insn->dst_reg, reg_type_str(env, dst_reg->type));

kernel/bpf/verifier.c

21038

if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg))

kernel/bpf/verifier.c

2104

bpf_vlog_reset(&env->log, head->log_pos);

kernel/bpf/verifier.c

21041

if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env,

kernel/bpf/verifier.c

21045

err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index);

kernel/bpf/verifier.c

21050

if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) {

kernel/bpf/verifier.c

21051

env->gotox_tmp_buf = iarray_realloc(env->gotox_tmp_buf,

kernel/bpf/verifier.c

21053

if (!env->gotox_tmp_buf)

kernel/bpf/verifier.c

21057

n = copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items);

kernel/bpf/verifier.c

21061

verbose(env, "register R%d doesn't point to any offset in map id=%d\n",

kernel/bpf/verifier.c

21067

other_branch = push_stack(env, env->gotox_tmp_buf->items[i],

kernel/bpf/verifier.c

21068

env->insn_idx, env->cur_state->speculative);

kernel/bpf/verifier.c

21072

env->insn_idx = env->gotox_tmp_buf->items[n-1];

kernel/bpf/verifier.c

21076

static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)

kernel/bpf/verifier.c

21079

struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx];

kernel/bpf/verifier.c

21083

err = check_alu_op(env, insn);

kernel/bpf/verifier.c

21093

err = check_load_mem(env, insn, false, is_ldsx, true, "ldx");

kernel/bpf/verifier.c

21098

err = check_atomic(env, insn);

kernel/bpf/verifier.c

21101

env->insn_idx++;

kernel/bpf/verifier.c

21106

verbose(env, "BPF_STX uses reserved fields\n");

kernel/bpf/verifier.c

21110

err = check_store_reg(env, insn, false);

kernel/bpf/verifier.c

21118

verbose(env, "BPF_ST uses reserved fields\n");

kernel/bpf/verifier.c

2112

env->head = elem;

kernel/bpf/verifier.c

21122

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

21126

dst_reg_type = cur_regs(env)[insn->dst_reg].type;

kernel/bpf/verifier.c

21129

err = check_mem_access(env, env->insn_idx, insn->dst_reg,

kernel/bpf/verifier.c

2113

env->stack_size--;

kernel/bpf/verifier.c

21135

err = save_aux_ptr_type(env, dst_reg_type, false);

kernel/bpf/verifier.c

21141

env->jmps_processed++;

kernel/bpf/verifier.c

21150

verbose(env, "BPF_CALL uses reserved fields\n");

kernel/bpf/verifier.c

21154

if (env->cur_state->active_locks) {

kernel/bpf/verifier.c

21159

verbose(env,

kernel/bpf/verifier.c

21165

err = check_func_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

21167

err = check_kfunc_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

21169

return process_bpf_exit_full(env, do_print_state, true);

kernel/bpf/verifier.c

21171

err = check_helper_call(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

21176

mark_reg_scratched(env, BPF_REG_0);

kernel/bpf/verifier.c

21181

verbose(env, "BPF_JA|BPF_X uses reserved fields\n");

kernel/bpf/verifier.c

21184

return check_indirect_jump(env, insn);

kernel/bpf/verifier.c

21192

verbose(env, "BPF_JA uses reserved fields\n");

kernel/bpf/verifier.c

21197

env->insn_idx += insn->off + 1;

kernel/bpf/verifier.c

21199

env->insn_idx += insn->imm + 1;

kernel/bpf/verifier.c

21207

verbose(env, "BPF_EXIT uses reserved fields\n");

kernel/bpf/verifier.c

21210

return process_bpf_exit_full(env, do_print_state, false);

kernel/bpf/verifier.c

21212

err = check_cond_jmp_op(env, insn, &env->insn_idx);

kernel/bpf/verifier.c

21220

err = check_ld_abs(env, insn);

kernel/bpf/verifier.c

21225

err = check_ld_imm(env, insn);

kernel/bpf/verifier.c

21229

env->insn_idx++;

kernel/bpf/verifier.c

21230

sanitize_mark_insn_seen(env);

kernel/bpf/verifier.c

21232

verbose(env, "invalid BPF_LD mode\n");

kernel/bpf/verifier.c

21236

verbose(env, "unknown insn class %d\n", class);

kernel/bpf/verifier.c

21240

env->insn_idx++;

kernel/bpf/verifier.c

21244

static int do_check(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

21246

bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);

kernel/bpf/verifier.c

21247

struct bpf_verifier_state *state = env->cur_state;

kernel/bpf/verifier.c

21248

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/verifier.c

21249

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

21259

env->cur_hist_ent = NULL;

kernel/bpf/verifier.c

21261

env->prev_insn_idx = prev_insn_idx;

kernel/bpf/verifier.c

21262

if (env->insn_idx >= insn_cnt) {

kernel/bpf/verifier.c

21263

verbose(env, "invalid insn idx %d insn_cnt %d\n",

kernel/bpf/verifier.c

21264

env->insn_idx, insn_cnt);

kernel/bpf/verifier.c

21268

insn = &insns[env->insn_idx];

kernel/bpf/verifier.c

21269

insn_aux = &env->insn_aux_data[env->insn_idx];

kernel/bpf/verifier.c

21271

if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {

kernel/bpf/verifier.c

21272

verbose(env,

kernel/bpf/verifier.c

21274

env->insn_processed);

kernel/bpf/verifier.c

21278

state->last_insn_idx = env->prev_insn_idx;

kernel/bpf/verifier.c

21279

state->insn_idx = env->insn_idx;

kernel/bpf/verifier.c

21281

if (is_prune_point(env, env->insn_idx)) {

kernel/bpf/verifier.c

21282

err = is_state_visited(env, env->insn_idx);

kernel/bpf/verifier.c

21287

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

21289

verbose(env, "\nfrom %d to %d%s: safe\n",

kernel/bpf/verifier.c

2129

static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21290

env->prev_insn_idx, env->insn_idx,

kernel/bpf/verifier.c

21291

env->cur_state->speculative ?

kernel/bpf/verifier.c

21294

verbose(env, "%d: safe\n", env->insn_idx);

kernel/bpf/verifier.c

21300

if (is_jmp_point(env, env->insn_idx)) {

kernel/bpf/verifier.c

21301

err = push_jmp_history(env, state, 0, 0);

kernel/bpf/verifier.c

21312

if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {

kernel/bpf/verifier.c

21313

verbose(env, "\nfrom %d to %d%s:",

kernel/bpf/verifier.c

21314

env->prev_insn_idx, env->insn_idx,

kernel/bpf/verifier.c

21315

env->cur_state->speculative ?

kernel/bpf/verifier.c

21317

print_verifier_state(env, state, state->curframe, true);

kernel/bpf/verifier.c

21321

if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

21322

if (verifier_state_scratched(env))

kernel/bpf/verifier.c

21323

print_insn_state(env, state, state->curframe);

kernel/bpf/verifier.c

21325

verbose_linfo(env, env->insn_idx, "; ");

kernel/bpf/verifier.c

21326

env->prev_log_pos = env->log.end_pos;

kernel/bpf/verifier.c

21327

verbose(env, "%d: ", env->insn_idx);

kernel/bpf/verifier.c

21328

verbose_insn(env, insn);

kernel/bpf/verifier.c

21329

env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;

kernel/bpf/verifier.c

2133

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

21330

env->prev_log_pos = env->log.end_pos;

kernel/bpf/verifier.c

21333

if (bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

21334

err = bpf_prog_offload_verify_insn(env, env->insn_idx,

kernel/bpf/verifier.c

21335

env->prev_insn_idx);

kernel/bpf/verifier.c

21340

sanitize_mark_insn_seen(env);

kernel/bpf/verifier.c

21341

prev_insn_idx = env->insn_idx;

kernel/bpf/verifier.c

21349

err = bpf_reset_stack_write_marks(env, env->insn_idx);

kernel/bpf/verifier.c

21352

err = do_check_insn(env, &do_print_state);

kernel/bpf/verifier.c

21354

marks_err = bpf_commit_stack_write_marks(env);

kernel/bpf/verifier.c

21392

BPF_OP(insn->code) != BPF_CALL, env,

kernel/bpf/verifier.c

21396

mark_verifier_state_scratched(env);

kernel/bpf/verifier.c

21397

err = update_branch_counts(env, env->cur_state);

kernel/bpf/verifier.c

21400

err = bpf_update_live_stack(env);

kernel/bpf/verifier.c

21403

err = pop_stack(env, &prev_insn_idx, &env->insn_idx,

kernel/bpf/verifier.c

2143

elem->next = env->head;

kernel/bpf/verifier.c

2144

elem->log_pos = env->log.end_pos;

kernel/bpf/verifier.c

2145

env->head = elem;

kernel/bpf/verifier.c

21450

static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)

kernel/bpf/verifier.c

21457

for (i = 0; i < env->used_btf_cnt; i++)

kernel/bpf/verifier.c

21458

if (env->used_btfs[i].btf == btf)

kernel/bpf/verifier.c

2146

env->stack_size++;

kernel/bpf/verifier.c

21461

if (env->used_btf_cnt >= MAX_USED_BTFS) {

kernel/bpf/verifier.c

21462

verbose(env, "The total number of btfs per program has reached the limit of %u\n",

kernel/bpf/verifier.c

21468

btf_mod = &env->used_btfs[env->used_btf_cnt];

kernel/bpf/verifier.c

21481

env->used_btf_cnt++;

kernel/bpf/verifier.c

21491

static int __check_pseudo_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21508

verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);

kernel/bpf/verifier.c

2151

if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {

kernel/bpf/verifier.c

21513

verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);

kernel/bpf/verifier.c

2152

verbose(env, "The sequence of %d jumps is too complex.\n",

kernel/bpf/verifier.c

21520

verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",

kernel/bpf/verifier.c

2153

env->stack_size);

kernel/bpf/verifier.c

21559

verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",

kernel/bpf/verifier.c

21574

static int check_pseudo_btf_id(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21586

verbose(env, "invalid module BTF object FD specified.\n");

kernel/bpf/verifier.c

21591

verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");

kernel/bpf/verifier.c

21598

err = __check_pseudo_btf_id(env, insn, aux, btf);

kernel/bpf/verifier.c

21604

return __add_used_btf(env, btf);

kernel/bpf/verifier.c

21627

static int check_map_prog_compatibility(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21636

verbose(env, "program's hash doesn't match map's excl_prog_hash\n");

kernel/bpf/verifier.c

21643

verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");

kernel/bpf/verifier.c

21650

verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");

kernel/bpf/verifier.c

21655

verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");

kernel/bpf/verifier.c

21662

verbose(env, "offload device mismatch between prog and map\n");

kernel/bpf/verifier.c

21667

verbose(env, "bpf_struct_ops map cannot be used in prog\n");

kernel/bpf/verifier.c

21694

verbose(env,

kernel/bpf/verifier.c

21700

bpf_cgroup_storage_assign(env->prog->aux, map)) {

kernel/bpf/verifier.c

21701

verbose(env, "only one cgroup storage of each type is allowed\n");

kernel/bpf/verifier.c

21706

if (env->prog->aux->arena) {

kernel/bpf/verifier.c

21707

verbose(env, "Only one arena per program\n");

kernel/bpf/verifier.c

21710

if (!env->allow_ptr_leaks || !env->bpf_capable) {

kernel/bpf/verifier.c

21711

verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");

kernel/bpf/verifier.c

21714

if (!env->prog->jit_requested) {

kernel/bpf/verifier.c

21715

verbose(env, "JIT is required to use arena\n");

kernel/bpf/verifier.c

21719

verbose(env, "JIT doesn't support arena\n");

kernel/bpf/verifier.c

21722

env->prog->aux->arena = (void *)map;

kernel/bpf/verifier.c

21723

if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {

kernel/bpf/verifier.c

21724

verbose(env, "arena's user address must be set via map_extra or mmap()\n");

kernel/bpf/verifier.c

21732

static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)

kernel/bpf/verifier.c

21737

for (i = 0; i < env->used_map_cnt; i++)

kernel/bpf/verifier.c

21738

if (env->used_maps[i] == map)

kernel/bpf/verifier.c

21741

if (env->used_map_cnt >= MAX_USED_MAPS) {

kernel/bpf/verifier.c

21742

verbose(env, "The total number of maps per program has reached the limit of %u\n",

kernel/bpf/verifier.c

21747

err = check_map_prog_compatibility(env, map, env->prog);

kernel/bpf/verifier.c

21751

if (env->prog->sleepable)

kernel/bpf/verifier.c

21761

env->used_maps[env->used_map_cnt++] = map;

kernel/bpf/verifier.c

21764

err = bpf_insn_array_init(map, env->prog);

kernel/bpf/verifier.c

21766

verbose(env, "Failed to properly initialize insn array\n");

kernel/bpf/verifier.c

21769

env->insn_array_maps[env->insn_array_map_cnt++] = map;

kernel/bpf/verifier.c

21772

return env->used_map_cnt - 1;

kernel/bpf/verifier.c

21779

static int add_used_map(struct bpf_verifier_env *env, int fd)

kernel/bpf/verifier.c

21786

verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);

kernel/bpf/verifier.c

21790

return __add_used_map(env, map);

kernel/bpf/verifier.c

21800

static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

21802

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

21803

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

21806

err = bpf_prog_calc_tag(env->prog);

kernel/bpf/verifier.c

21814

verbose(env, "BPF_LDX uses reserved fields\n");

kernel/bpf/verifier.c

21828

verbose(env, "invalid bpf_ld_imm64 insn\n");

kernel/bpf/verifier.c

21837

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

21838

err = check_pseudo_btf_id(env, insn, aux);

kernel/bpf/verifier.c

21845

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

21863

verbose(env, "unrecognized bpf_ld_imm64 insn\n");

kernel/bpf/verifier.c

21870

if (bpfptr_is_null(env->fd_array)) {

kernel/bpf/verifier.c

21871

verbose(env, "fd_idx without fd_array is invalid\n");

kernel/bpf/verifier.c

21874

if (copy_from_bpfptr_offset(&fd, env->fd_array,

kernel/bpf/verifier.c

21884

map_idx = add_used_map(env, fd);

kernel/bpf/verifier.c

21887

map = env->used_maps[map_idx];

kernel/bpf/verifier.c

21889

aux = &env->insn_aux_data[i];

kernel/bpf/verifier.c

21899

verbose(env, "no direct value access support for this map type\n");

kernel/bpf/verifier.c

21905

verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",

kernel/bpf/verifier.c

21925

verbose(env, "unknown opcode %02x\n", insn->code);

kernel/bpf/verifier.c

21938

static void release_maps(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

21940

__bpf_free_used_maps(env->prog->aux, env->used_maps,

kernel/bpf/verifier.c

21941

env->used_map_cnt);

kernel/bpf/verifier.c

21945

static void release_btfs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

21947

__bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);

kernel/bpf/verifier.c

21951

static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

21953

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

21954

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

21970

static void adjust_insn_aux_data(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

21973

struct bpf_insn_aux_data *data = env->insn_aux_data;

kernel/bpf/verifier.c

21999

static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/verifier.c

22006

for (i = 0; i <= env->subprog_cnt; i++) {

kernel/bpf/verifier.c

22007

if (env->subprog_info[i].start <= off)

kernel/bpf/verifier.c

22009

env->subprog_info[i].start += len - 1;

kernel/bpf/verifier.c

22013

static void release_insn_arrays(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22017

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/verifier.c

22018

bpf_insn_array_release(env->insn_array_maps[i]);

kernel/bpf/verifier.c

22021

static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/verifier.c

22028

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/verifier.c

22029

bpf_insn_array_adjust(env->insn_array_maps[i], off, len);

kernel/bpf/verifier.c

22032

static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)

kernel/bpf/verifier.c

22036

for (i = 0; i < env->insn_array_map_cnt; i++)

kernel/bpf/verifier.c

22037

bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);

kernel/bpf/verifier.c

22054

static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,

kernel/bpf/verifier.c

22061

new_data = vrealloc(env->insn_aux_data,

kernel/bpf/verifier.c

22062

array_size(env->prog->len + len - 1,

kernel/bpf/verifier.c

22068

env->insn_aux_data = new_data;

kernel/bpf/verifier.c

22071

new_prog = bpf_patch_insn_single(env->prog, off, patch, len);

kernel/bpf/verifier.c

22074

verbose(env,

kernel/bpf/verifier.c

22076

env->insn_aux_data[off].orig_idx);

kernel/bpf/verifier.c

22079

adjust_insn_aux_data(env, new_prog, off, len);

kernel/bpf/verifier.c

22080

adjust_subprog_starts(env, off, len);

kernel/bpf/verifier.c

22081

adjust_insn_arrays(env, off, len);

kernel/bpf/verifier.c

22124

static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

22130

for (i = 0; i < env->subprog_cnt; i++)

kernel/bpf/verifier.c

22131

if (env->subprog_info[i].start >= off)

kernel/bpf/verifier.c

22134

for (j = i; j < env->subprog_cnt; j++)

kernel/bpf/verifier.c

22135

if (env->subprog_info[j].start >= off + cnt)

kernel/bpf/verifier.c

22140

if (env->subprog_info[j].start != off + cnt)

kernel/bpf/verifier.c

22144

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

22148

move = env->subprog_cnt + 1 - j;

kernel/bpf/verifier.c

22150

memmove(env->subprog_info + i,

kernel/bpf/verifier.c

22151

env->subprog_info + j,

kernel/bpf/verifier.c

22152

sizeof(*env->subprog_info) * move);

kernel/bpf/verifier.c

22153

env->subprog_cnt -= j - i;

kernel/bpf/verifier.c

22169

if (env->subprog_info[i].start == off)

kernel/bpf/verifier.c

22174

for (; i <= env->subprog_cnt; i++)

kernel/bpf/verifier.c

22175

env->subprog_info[i].start -= cnt;

kernel/bpf/verifier.c

22180

static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,

kernel/bpf/verifier.c

22183

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

2221

static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

22230

for (i = 0; i <= env->subprog_cnt; i++)

kernel/bpf/verifier.c

22231

if (env->subprog_info[i].linfo_idx > l_off) {

kernel/bpf/verifier.c

22235

if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)

kernel/bpf/verifier.c

22236

env->subprog_info[i].linfo_idx -= l_cnt;

kernel/bpf/verifier.c

22238

env->subprog_info[i].linfo_idx = l_off;

kernel/bpf/verifier.c

22247

static void clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)

kernel/bpf/verifier.c

22249

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

22250

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/verifier.c

22265

static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)

kernel/bpf/verifier.c

22267

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

22268

unsigned int orig_prog_len = env->prog->len;

kernel/bpf/verifier.c

22271

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/verifier.c

22272

bpf_prog_offload_remove_insns(env, off, cnt);

kernel/bpf/verifier.c

22275

clear_insn_aux_data(env, off, cnt);

kernel/bpf/verifier.c

22277

err = bpf_remove_insns(env->prog, off, cnt);

kernel/bpf/verifier.c

2228

reg->precise = !env->bpf_capable;

kernel/bpf/verifier.c

22281

err = adjust_subprog_starts_after_remove(env, off, cnt);

kernel/bpf/verifier.c

22285

err = bpf_adj_linfo_after_remove(env, off, cnt);

kernel/bpf/verifier.c

22289

adjust_insn_arrays_after_remove(env, off, cnt);

kernel/bpf/verifier.c

22308

static void sanitize_dead_code(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

2231

static void mark_reg_known_zero(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

22310

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

22312

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

22313

const int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

22338

static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22340

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

22342

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

22343

const int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

2235

verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);

kernel/bpf/verifier.c

22357

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/verifier.c

22358

bpf_prog_offload_replace_insn(env, i, &ja);

kernel/bpf/verifier.c

22364

static int opt_remove_dead_code(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22366

struct bpf_insn_aux_data *aux_data = env->insn_aux_data;

kernel/bpf/verifier.c

22367

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

22379

err = verifier_remove_insns(env, i, j);

kernel/bpf/verifier.c

2238

__mark_reg_not_init(env, regs + regno);

kernel/bpf/verifier.c

22382

insn_cnt = env->prog->len;

kernel/bpf/verifier.c

22391

static int opt_remove_nops(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22393

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

22394

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

22405

err = verifier_remove_insns(env, i, 1);

kernel/bpf/verifier.c

22416

static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

22421

struct bpf_insn *zext_patch = env->insn_buf;

kernel/bpf/verifier.c

22422

struct bpf_insn *rnd_hi32_patch = &env->insn_buf[2];

kernel/bpf/verifier.c

22423

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

22424

int i, patch_len, delta = 0, len = env->prog->len;

kernel/bpf/verifier.c

22425

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/verifier.c

22495

if (verifier_bug_if(load_reg == -1, env,

kernel/bpf/verifier.c

22505

new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);

kernel/bpf/verifier.c

22508

env->prog = new_prog;

kernel/bpf/verifier.c

22510

aux = env->insn_aux_data;

kernel/bpf/verifier.c

22522

static int convert_ctx_accesses(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22524

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/verifier.c

22525

const struct bpf_verifier_ops *ops = env->ops;

kernel/bpf/verifier.c

22527

const int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

22528

struct bpf_insn *epilogue_buf = env->epilogue_buf;

kernel/bpf/verifier.c

22529

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/verifier.c

22538

epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,

kernel/bpf/verifier.c

22541

verifier_bug(env, "epilogue is too long");

kernel/bpf/verifier.c

22549

insn_buf[cnt++] = env->prog->insnsi[0];

kernel/bpf/verifier.c

22550

new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);

kernel/bpf/verifier.c

22553

env->prog = new_prog;

kernel/bpf/verifier.c

22556

ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);

kernel/bpf/verifier.c

22562

if (ops->gen_prologue || env->seen_direct_write) {

kernel/bpf/verifier.c

22564

verifier_bug(env, "gen_prologue is null");

kernel/bpf/verifier.c

22567

cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,

kernel/bpf/verifier.c

22568

env->prog);

kernel/bpf/verifier.c

22570

verifier_bug(env, "prologue is too long");

kernel/bpf/verifier.c

22573

new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);

kernel/bpf/verifier.c

22577

env->prog = new_prog;

kernel/bpf/verifier.c

22580

ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);

kernel/bpf/verifier.c

22587

WARN_ON(adjust_jmp_off(env->prog, 0, delta));

kernel/bpf/verifier.c

22589

if (bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/verifier.c

22592

insn = env->prog->insnsi + delta;

kernel/bpf/verifier.c

22598

if (env->insn_aux_data[i + delta].nospec) {

kernel/bpf/verifier.c

22599

WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);

kernel/bpf/verifier.c

22605

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

22610

env->prog = new_prog;

kernel/bpf/verifier.c

22641

env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {

kernel/bpf/verifier.c

22643

env->prog->aux->num_exentries++;

kernel/bpf/verifier.c

22669

env->insn_aux_data[i + delta].nospec_result) {

kernel/bpf/verifier.c

22678

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

22683

env->prog = new_prog;

kernel/bpf/verifier.c

22688

switch ((int)env->insn_aux_data[i + delta].ptr_type) {

kernel/bpf/verifier.c

22721

env->prog->aux->num_exentries++;

kernel/bpf/verifier.c

22727

verbose(env, "sign extending loads from arena are not supported yet\n");

kernel/bpf/verifier.c

22734

env->prog->aux->num_exentries++;

kernel/bpf/verifier.c

22740

ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;

kernel/bpf/verifier.c

22756

verifier_bug(env, "narrow ctx access misconfigured");

kernel/bpf/verifier.c

22771

cnt = convert_ctx_access(type, insn, insn_buf, env->prog,

kernel/bpf/verifier.c

22775

verifier_bug(env, "error during ctx access conversion (%d)", cnt);

kernel/bpf/verifier.c

22783

verifier_bug(env, "narrow ctx load misconfigured");

kernel/bpf/verifier.c

22808

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

22815

env->prog = new_prog;

kernel/bpf/verifier.c

22822

static int jit_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

22824

struct bpf_prog *prog = env->prog, **func, *tmp;

kernel/bpf/verifier.c

22832

if (env->subprog_cnt <= 1)

kernel/bpf/verifier.c

22843

subprog = find_subprog(env, i + insn->imm + 1);

kernel/bpf/verifier.c

22844

if (verifier_bug_if(subprog < 0, env, "No program to jit at insn %d",

kernel/bpf/verifier.c

22854

env->insn_aux_data[i].call_imm = insn->imm;

kernel/bpf/verifier.c

22877

func = kzalloc_objs(prog, env->subprog_cnt);

kernel/bpf/verifier.c

22881

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

22883

subprog_end = env->subprog_info[i + 1].start;

kernel/bpf/verifier.c

22922

func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;

kernel/bpf/verifier.c

22923

if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)

kernel/bpf/verifier.c

22933

func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;

kernel/bpf/verifier.c

22935

func[i]->aux->used_maps = env->used_maps;

kernel/bpf/verifier.c

22936

func[i]->aux->used_map_cnt = env->used_map_cnt;

kernel/bpf/verifier.c

22955

func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;

kernel/bpf/verifier.c

22956

func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;

kernel/bpf/verifier.c

22957

func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;

kernel/bpf/verifier.c

22958

func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;

kernel/bpf/verifier.c

22960

func[i]->aux->exception_boundary = env->seen_exception;

kernel/bpf/verifier.c

22981

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

23008

func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;

kernel/bpf/verifier.c

23009

func[i]->aux->real_func_cnt = env->subprog_cnt;

kernel/bpf/verifier.c

23011

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

23015

verbose(env, "JIT doesn't support bpf-to-bpf calls\n");

kernel/bpf/verifier.c

23026

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

23035

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

23041

for (i = 1; i < env->subprog_cnt; i++)

kernel/bpf/verifier.c

23050

insn[0].imm = env->insn_aux_data[i].call_imm;

kernel/bpf/verifier.c

23057

insn->off = env->insn_aux_data[i].call_imm;

kernel/bpf/verifier.c

23058

subprog = find_subprog(env, i + insn->off + 1);

kernel/bpf/verifier.c

23068

prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;

kernel/bpf/verifier.c

23069

prog->aux->real_func_cnt = env->subprog_cnt;

kernel/bpf/verifier.c

23070

prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;

kernel/bpf/verifier.c

23087

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

23102

insn->imm = env->insn_aux_data[i].call_imm;

kernel/bpf/verifier.c

23108

static int fixup_call_args(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

23111

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

23118

if (env->prog->jit_requested &&

kernel/bpf/verifier.c

23119

!bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

23120

err = jit_subprogs(env);

kernel/bpf/verifier.c

23128

verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");

kernel/bpf/verifier.c

23131

if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {

kernel/bpf/verifier.c

23135

verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");

kernel/bpf/verifier.c

23143

verbose(env, "callbacks are not allowed in non-JITed programs\n");

kernel/bpf/verifier.c

23149

depth = get_callee_stack_depth(env, insn, i);

kernel/bpf/verifier.c

23160

static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx)

kernel/bpf/verifier.c

23162

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

23179

seen_direct_write = env->seen_direct_write;

kernel/bpf/verifier.c

23180

is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);

kernel/bpf/verifier.c

23188

env->seen_direct_write = seen_direct_write;

kernel/bpf/verifier.c

23196

if (!env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

23199

if (env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

23202

if (env->insn_aux_data[insn_idx].non_sleepable)

kernel/bpf/verifier.c

23226

static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

23233

verbose(env, "invalid kernel function call not eliminated in verifier pass\n");

kernel/bpf/verifier.c

23243

desc = find_kfunc_desc(env->prog, insn->imm, insn->off);

kernel/bpf/verifier.c

23245

verifier_bug(env, "kernel function descriptor not found for func_id %u",

kernel/bpf/verifier.c

23250

err = specialize_kfunc(env, desc, insn_idx);

kernel/bpf/verifier.c

23259

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

23261

u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;

kernel/bpf/verifier.c

23264

verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

23277

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

23281

verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

23288

verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

23300

struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;

kernel/bpf/verifier.c

23311

verifier_bug(env, "kptr_struct_meta expected at insn_idx %d",

kernel/bpf/verifier.c

23316

__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,

kernel/bpf/verifier.c

23323

env->prog->expected_attach_type == BPF_TRACE_FSESSION) {

kernel/bpf/verifier.c

23336

env->prog->expected_attach_type == BPF_TRACE_FSESSION) {

kernel/bpf/verifier.c

23354

if (env->insn_aux_data[insn_idx].arg_prog) {

kernel/bpf/verifier.c

23355

u32 regno = env->insn_aux_data[insn_idx].arg_prog;

kernel/bpf/verifier.c

23356

struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };

kernel/bpf/verifier.c

23368

static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)

kernel/bpf/verifier.c

23370

struct bpf_subprog_info *info = env->subprog_info;

kernel/bpf/verifier.c

23371

int cnt = env->subprog_cnt;

kernel/bpf/verifier.c

23375

if (env->hidden_subprog_cnt) {

kernel/bpf/verifier.c

23376

verifier_bug(env, "only one hidden subprog supported");

kernel/bpf/verifier.c

23383

prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);

kernel/bpf/verifier.c

23386

env->prog = prog;

kernel/bpf/verifier.c

23389

env->subprog_cnt++;

kernel/bpf/verifier.c

23390

env->hidden_subprog_cnt++;

kernel/bpf/verifier.c

23397

static int do_misc_fixups(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

23399

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

23407

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/verifier.c

23411

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/verifier.c

23415

if (env->seen_exception && !env->exception_callback_subprog) {

kernel/bpf/verifier.c

23418

*patch++ = env->prog->insnsi[insn_cnt - 1];

kernel/bpf/verifier.c

23421

ret = add_hidden_subprog(env, insn_buf, patch - insn_buf);

kernel/bpf/verifier.c

23424

prog = env->prog;

kernel/bpf/verifier.c

23427

env->exception_callback_subprog = env->subprog_cnt - 1;

kernel/bpf/verifier.c

23429

mark_subprog_exc_cb(env, env->exception_callback_subprog);

kernel/bpf/verifier.c

23435

(((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {

kernel/bpf/verifier.c

23445

if (env->insn_aux_data[i + delta].needs_zext)

kernel/bpf/verifier.c

23468

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23473

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23559

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23564

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23589

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23594

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23603

cnt = env->ops->gen_ld_abs(insn, insn_buf);

kernel/bpf/verifier.c

23605

verifier_bug(env, "%d insns generated for ld_abs", cnt);

kernel/bpf/verifier.c

23609

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23614

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23628

aux = &env->insn_aux_data[i + delta];

kernel/bpf/verifier.c

23662

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23667

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23707

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23712

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23728

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23733

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23743

ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);

kernel/bpf/verifier.c

23749

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23754

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23776

if (!allow_tail_call_in_subprogs(env))

kernel/bpf/verifier.c

23788

aux = &env->insn_aux_data[i + delta];

kernel/bpf/verifier.c

23789

if (env->bpf_capable && !prog->blinding_requested &&

kernel/bpf/verifier.c

23803

verbose(env, "adding tail call poke descriptor failed\n");

kernel/bpf/verifier.c

23821

verbose(env, "tail_call abusing map_ptr\n");

kernel/bpf/verifier.c

23834

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23839

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23867

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23872

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23878

if (env->insn_aux_data[i + delta].non_sleepable)

kernel/bpf/verifier.c

23885

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23890

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23896

if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {

kernel/bpf/verifier.c

23904

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

23909

env->prog = prog = new_prog;

kernel/bpf/verifier.c

23928

aux = &env->insn_aux_data[i + delta];

kernel/bpf/verifier.c

23940

verifier_bug(env, "%d insns generated for map lookup", cnt);

kernel/bpf/verifier.c

23944

new_prog = bpf_patch_insn_data(env, i + delta,

kernel/bpf/verifier.c

23950

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24027

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,

kernel/bpf/verifier.c

24033

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24041

verifier_inlines_helper_call(env, insn->imm)) {

kernel/bpf/verifier.c

24056

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24061

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24068

verifier_inlines_helper_call(env, insn->imm)) {

kernel/bpf/verifier.c

24074

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24079

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24108

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24113

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24138

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24143

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24164

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24169

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24180

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);

kernel/bpf/verifier.c

24184

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24235

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24240

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24253

new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);

kernel/bpf/verifier.c

24258

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24263

fn = env->ops->get_func_proto(insn->imm, env->prog);

kernel/bpf/verifier.c

24268

verifier_bug(env,

kernel/bpf/verifier.c

24281

verbose(env, "stack size %d(extra %d) is too large\n",

kernel/bpf/verifier.c

24293

env->prog->aux->stack_depth = subprogs[0].stack_depth;

kernel/bpf/verifier.c

24294

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

24304

verifier_bug(env, "stack_slots supports may_goto only");

kernel/bpf/verifier.c

24319

insn_buf[cnt++] = env->prog->insnsi[subprog_start];

kernel/bpf/verifier.c

24321

new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);

kernel/bpf/verifier.c

24324

env->prog = prog = new_prog;

kernel/bpf/verifier.c

24331

WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));

kernel/bpf/verifier.c

24340

verifier_bug(env, "poke tab is misconfigured");

kernel/bpf/verifier.c

24346

verbose(env, "tracking tail call prog failed\n");

kernel/bpf/verifier.c

24351

ret = sort_kfunc_descs_by_imm_off(env);

kernel/bpf/verifier.c

24358

static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

24371

struct bpf_insn *insn_buf = env->insn_buf;

kernel/bpf/verifier.c

24420

new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);

kernel/bpf/verifier.c

24425

callback_start = env->subprog_info[callback_subprogno].start;

kernel/bpf/verifier.c

24450

static int optimize_bpf_loop(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24452

struct bpf_subprog_info *subprogs = env->subprog_info;

kernel/bpf/verifier.c

24454

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

24455

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

24462

&env->insn_aux_data[i + delta].loop_inline_state;

kernel/bpf/verifier.c

24468

new_prog = inline_bpf_loop(env,

kernel/bpf/verifier.c

24477

env->prog = new_prog;

kernel/bpf/verifier.c

24490

env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;

kernel/bpf/verifier.c

24498

static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24500

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

24501

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

24502

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

24503

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

24529

static void free_states(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24536

free_verifier_state(env->cur_state, true);

kernel/bpf/verifier.c

24537

env->cur_state = NULL;

kernel/bpf/verifier.c

24538

while (!pop_stack(env, NULL, NULL, false));

kernel/bpf/verifier.c

24540

list_for_each_safe(pos, tmp, &env->free_list) {

kernel/bpf/verifier.c

24545

INIT_LIST_HEAD(&env->free_list);

kernel/bpf/verifier.c

24547

for (i = 0; i < env->scc_cnt; ++i) {

kernel/bpf/verifier.c

24548

info = env->scc_info[i];

kernel/bpf/verifier.c

24554

env->scc_info[i] = NULL;

kernel/bpf/verifier.c

24557

if (!env->explored_states)

kernel/bpf/verifier.c

24560

for (i = 0; i < state_htab_size(env); i++) {

kernel/bpf/verifier.c

24561

head = &env->explored_states[i];

kernel/bpf/verifier.c

24568

INIT_LIST_HEAD(&env->explored_states[i]);

kernel/bpf/verifier.c

24572

static int do_check_common(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

24574

bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);

kernel/bpf/verifier.c

24575

struct bpf_subprog_info *sub = subprog_info(env, subprog);

kernel/bpf/verifier.c

24576

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

24581

env->prev_linfo = NULL;

kernel/bpf/verifier.c

24582

env->pass_cnt++;

kernel/bpf/verifier.c

24590

state->in_sleepable = env->prog->sleepable;

kernel/bpf/verifier.c

24596

env->cur_state = state;

kernel/bpf/verifier.c

24597

init_func_state(env, state->frame[0],

kernel/bpf/verifier.c

24601

state->first_insn_idx = env->subprog_info[subprog].start;

kernel/bpf/verifier.c

24605

if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {

kernel/bpf/verifier.c

24606

const char *sub_name = subprog_name(env, subprog);

kernel/bpf/verifier.c

24610

if (env->log.level & BPF_LOG_LEVEL)

kernel/bpf/verifier.c

24611

verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);

kernel/bpf/verifier.c

24612

ret = btf_prepare_func_args(env, subprog);

kernel/bpf/verifier.c

24616

if (subprog_is_exc_cb(env, subprog)) {

kernel/bpf/verifier.c

24623

verbose(env, "exception cb only supports single integer argument\n");

kernel/bpf/verifier.c

24634

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

24637

mark_reg_unknown(env, regs, i);

kernel/bpf/verifier.c

24640

__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);

kernel/bpf/verifier.c

24645

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

24648

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

24657

mark_reg_known_zero(env, regs, i);

kernel/bpf/verifier.c

24660

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

24663

mark_reg_unknown(env, regs, i);

kernel/bpf/verifier.c

24665

verifier_bug(env, "unhandled arg#%d type %d",

kernel/bpf/verifier.c

24676

if (env->prog->aux->func_info_aux) {

kernel/bpf/verifier.c

24677

ret = btf_prepare_func_args(env, 0);

kernel/bpf/verifier.c

24679

env->prog->aux->func_info_aux[0].unreliable = true;

kernel/bpf/verifier.c

24684

mark_reg_known_zero(env, regs, BPF_REG_1);

kernel/bpf/verifier.c

24688

if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {

kernel/bpf/verifier.c

24691

acquire_reference(env, 0) : 0;

kernel/bpf/verifier.c

24694

ret = do_check(env);

kernel/bpf/verifier.c

24697

bpf_vlog_reset(&env->log, 0);

kernel/bpf/verifier.c

24698

free_states(env);

kernel/bpf/verifier.c

24722

static int do_check_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24724

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

24732

if (env->exception_callback_subprog)

kernel/bpf/verifier.c

24733

subprog_aux(env, env->exception_callback_subprog)->called = true;

kernel/bpf/verifier.c

24737

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

24738

if (!subprog_is_global(env, i))

kernel/bpf/verifier.c

24741

sub_aux = subprog_aux(env, i);

kernel/bpf/verifier.c

24745

env->insn_idx = env->subprog_info[i].start;

kernel/bpf/verifier.c

24746

WARN_ON_ONCE(env->insn_idx == 0);

kernel/bpf/verifier.c

24747

ret = do_check_common(env, i);

kernel/bpf/verifier.c

24750

} else if (env->log.level & BPF_LOG_LEVEL) {

kernel/bpf/verifier.c

24751

verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",

kernel/bpf/verifier.c

24752

i, subprog_name(env, i));

kernel/bpf/verifier.c

24772

static int do_check_main(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24776

env->insn_idx = 0;

kernel/bpf/verifier.c

24777

ret = do_check_common(env, 0);

kernel/bpf/verifier.c

24779

env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;

kernel/bpf/verifier.c

24784

static void print_verification_stats(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24788

if (env->log.level & BPF_LOG_STATS) {

kernel/bpf/verifier.c

24789

verbose(env, "verification time %lld usec\n",

kernel/bpf/verifier.c

24790

div_u64(env->verification_time, 1000));

kernel/bpf/verifier.c

24791

verbose(env, "stack depth ");

kernel/bpf/verifier.c

24792

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

24793

u32 depth = env->subprog_info[i].stack_depth;

kernel/bpf/verifier.c

24795

verbose(env, "%d", depth);

kernel/bpf/verifier.c

24796

if (i + 1 < env->subprog_cnt)

kernel/bpf/verifier.c

24797

verbose(env, "+");

kernel/bpf/verifier.c

24799

verbose(env, "\n");

kernel/bpf/verifier.c

24801

verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "

kernel/bpf/verifier.c

24803

env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,

kernel/bpf/verifier.c

24804

env->max_states_per_insn, env->total_states,

kernel/bpf/verifier.c

24805

env->peak_states, env->longest_mark_read_walk);

kernel/bpf/verifier.c

24817

static int check_struct_ops_btf_id(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

24823

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

24831

verbose(env, "struct ops programs must have a GPL compatible license\n");

kernel/bpf/verifier.c

24841

env->attach_btf_mod = btf_try_get_module(btf);

kernel/bpf/verifier.c

24842

if (!env->attach_btf_mod) {

kernel/bpf/verifier.c

24843

verbose(env, "struct_ops module %s is not found\n",

kernel/bpf/verifier.c

24852

verbose(env, "attach_btf_id %u is not a supported struct\n",

kernel/bpf/verifier.c

24861

verbose(env, "attach to invalid member idx %u of struct %s\n",

kernel/bpf/verifier.c

24871

verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",

kernel/bpf/verifier.c

24879

verbose(env, "attach to unsupported member %s of struct %s\n",

kernel/bpf/verifier.c

24888

verbose(env, "attach to unsupported member %s of struct %s\n",

kernel/bpf/verifier.c

24895

verbose(env, "Private stack not supported by jit\n");

kernel/bpf/verifier.c

24909

for (i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

24910

if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {

kernel/bpf/verifier.c

24911

verbose(env, "program with __ref argument cannot tail call\n");

kernel/bpf/verifier.c

24921

env->ops = st_ops->verifier_ops;

kernel/bpf/verifier.c

25361

static int check_attach_btf_id(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

25363

struct bpf_prog *prog = env->prog;

kernel/bpf/verifier.c

25375

verbose(env, "Syscall programs can only be sleepable\n");

kernel/bpf/verifier.c

25380

verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");

kernel/bpf/verifier.c

25385

return check_struct_ops_btf_id(env);

kernel/bpf/verifier.c

25392

ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);

kernel/bpf/verifier.c

25401

env->ops = bpf_verifier_ops[tgt_prog->type];

kernel/bpf/verifier.c

25423

ret = bpf_lsm_verify_prog(&env->log, prog);

kernel/bpf/verifier.c

25428

verbose(env, "Attaching tracing programs to function '%s' is rejected.\n",

kernel/bpf/verifier.c

25435

verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n",

kernel/bpf/verifier.c

25468

static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)

kernel/bpf/verifier.c

25477

err = __add_used_map(env, map);

kernel/bpf/verifier.c

25486

return __add_used_btf(env, btf);

kernel/bpf/verifier.c

25489

verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);

kernel/bpf/verifier.c

25493

static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)

kernel/bpf/verifier.c

25500

env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);

kernel/bpf/verifier.c

25512

verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);

kernel/bpf/verifier.c

25517

if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))

kernel/bpf/verifier.c

25520

ret = add_fd_from_fd_array(env, fd);

kernel/bpf/verifier.c

25540

static void compute_insn_live_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

25659

if (get_call_summary(env, insn, &cs))

kernel/bpf/verifier.c

25683

static int compute_live_registers(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

25685

struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;

kernel/bpf/verifier.c

25686

struct bpf_insn *insns = env->prog->insnsi;

kernel/bpf/verifier.c

25688

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

25718

compute_insn_live_regs(env, &insns[i], &state[i]);

kernel/bpf/verifier.c

25723

for (i = 0; i < env->cfg.cur_postorder; ++i) {

kernel/bpf/verifier.c

25724

int insn_idx = env->cfg.insn_postorder[i];

kernel/bpf/verifier.c

25730

succ = bpf_insn_successors(env, insn_idx);

kernel/bpf/verifier.c

25745

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

25746

verbose(env, "Live regs before insn:\n");

kernel/bpf/verifier.c

25748

if (env->insn_aux_data[i].scc)

kernel/bpf/verifier.c

25749

verbose(env, "%3d ", env->insn_aux_data[i].scc);

kernel/bpf/verifier.c

25751

verbose(env, " ");

kernel/bpf/verifier.c

25752

verbose(env, "%3d: ", i);

kernel/bpf/verifier.c

25755

verbose(env, "%d", j);

kernel/bpf/verifier.c

25757

verbose(env, ".");

kernel/bpf/verifier.c

25758

verbose(env, " ");

kernel/bpf/verifier.c

25759

verbose_insn(env, &insns[i]);

kernel/bpf/verifier.c

25777

static int compute_scc(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

25781

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

25782

const u32 insn_cnt = env->prog->len;

kernel/bpf/verifier.c

25896

succ = bpf_insn_successors(env, w);

kernel/bpf/verifier.c

25925

if (bpf_calls_callback(env, w)) /* implicit loop? */

kernel/bpf/verifier.c

25939

env->scc_info = kvzalloc_objs(*env->scc_info, next_scc_id,

kernel/bpf/verifier.c

25941

if (!env->scc_info) {

kernel/bpf/verifier.c

25945

env->scc_cnt = next_scc_id;

kernel/bpf/verifier.c

25957

struct bpf_verifier_env *env;

kernel/bpf/verifier.c

25971

env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT);

kernel/bpf/verifier.c

25972

if (!env)

kernel/bpf/verifier.c

25975

env->bt.env = env;

kernel/bpf/verifier.c

25978

env->insn_aux_data =

kernel/bpf/verifier.c

25981

if (!env->insn_aux_data)

kernel/bpf/verifier.c

25984

env->insn_aux_data[i].orig_idx = i;

kernel/bpf/verifier.c

25985

env->succ = iarray_realloc(NULL, 2);

kernel/bpf/verifier.c

25986

if (!env->succ)

kernel/bpf/verifier.c

25988

env->prog = *prog;

kernel/bpf/verifier.c

25989

env->ops = bpf_verifier_ops[env->prog->type];

kernel/bpf/verifier.c

25991

env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);

kernel/bpf/verifier.c

25992

env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);

kernel/bpf/verifier.c

25993

env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);

kernel/bpf/verifier.c

25994

env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);

kernel/bpf/verifier.c

25995

env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);

kernel/bpf/verifier.c

26006

ret = bpf_vlog_init(&env->log, attr->log_level,

kernel/bpf/verifier.c

26012

ret = process_fd_array(env, attr, uattr);

kernel/bpf/verifier.c

26016

mark_verifier_state_clean(env);

kernel/bpf/verifier.c

26020

verbose(env, "in-kernel BTF is malformed\n");

kernel/bpf/verifier.c

26025

env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);

kernel/bpf/verifier.c

26027

env->strict_alignment = true;

kernel/bpf/verifier.c

26029

env->strict_alignment = false;

kernel/bpf/verifier.c

26032

env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;

kernel/bpf/verifier.c

26033

env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;

kernel/bpf/verifier.c

26035

env->explored_states = kvzalloc_objs(struct list_head,

kernel/bpf/verifier.c

26036

state_htab_size(env),

kernel/bpf/verifier.c

26039

if (!env->explored_states)

kernel/bpf/verifier.c

26042

for (i = 0; i < state_htab_size(env); i++)

kernel/bpf/verifier.c

26043

INIT_LIST_HEAD(&env->explored_states[i]);

kernel/bpf/verifier.c

26044

INIT_LIST_HEAD(&env->free_list);

kernel/bpf/verifier.c

26046

ret = check_btf_info_early(env, attr, uattr);

kernel/bpf/verifier.c

26050

ret = add_subprog_and_kfunc(env);

kernel/bpf/verifier.c

26054

ret = check_subprogs(env);

kernel/bpf/verifier.c

26058

ret = check_btf_info(env, attr, uattr);

kernel/bpf/verifier.c

26062

ret = resolve_pseudo_ldimm64(env);

kernel/bpf/verifier.c

26066

if (bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

26067

ret = bpf_prog_offload_verifier_prep(env->prog);

kernel/bpf/verifier.c

26072

ret = check_cfg(env);

kernel/bpf/verifier.c

26076

ret = compute_postorder(env);

kernel/bpf/verifier.c

26080

ret = bpf_stack_liveness_init(env);

kernel/bpf/verifier.c

26084

ret = check_attach_btf_id(env);

kernel/bpf/verifier.c

26088

ret = compute_scc(env);

kernel/bpf/verifier.c

26092

ret = compute_live_registers(env);

kernel/bpf/verifier.c

26096

ret = mark_fastcall_patterns(env);

kernel/bpf/verifier.c

26100

ret = do_check_main(env);

kernel/bpf/verifier.c

26101

ret = ret ?: do_check_subprogs(env);

kernel/bpf/verifier.c

26103

if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))

kernel/bpf/verifier.c

26104

ret = bpf_prog_offload_finalize(env);

kernel/bpf/verifier.c

26107

kvfree(env->explored_states);

kernel/bpf/verifier.c

26113

ret = remove_fastcall_spills_fills(env);

kernel/bpf/verifier.c

26116

ret = check_max_stack_depth(env);

kernel/bpf/verifier.c

26120

ret = optimize_bpf_loop(env);

kernel/bpf/verifier.c

26124

opt_hard_wire_dead_code_branches(env);

kernel/bpf/verifier.c

26126

ret = opt_remove_dead_code(env);

kernel/bpf/verifier.c

26128

ret = opt_remove_nops(env);

kernel/bpf/verifier.c

26131

sanitize_dead_code(env);

kernel/bpf/verifier.c

26136

ret = convert_ctx_accesses(env);

kernel/bpf/verifier.c

26139

ret = do_misc_fixups(env);

kernel/bpf/verifier.c

26144

if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {

kernel/bpf/verifier.c

26145

ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);

kernel/bpf/verifier.c

26146

env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret

kernel/bpf/verifier.c

26151

ret = fixup_call_args(env);

kernel/bpf/verifier.c

26153

env->verification_time = ktime_get_ns() - start_time;

kernel/bpf/verifier.c

26154

print_verification_stats(env);

kernel/bpf/verifier.c

26155

env->prog->aux->verified_insns = env->insn_processed;

kernel/bpf/verifier.c

26158

err = bpf_vlog_finalize(&env->log, &log_true_size);

kernel/bpf/verifier.c

26172

if (env->used_map_cnt) {

kernel/bpf/verifier.c

26174

env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0],

kernel/bpf/verifier.c

26175

env->used_map_cnt,

kernel/bpf/verifier.c

26178

if (!env->prog->aux->used_maps) {

kernel/bpf/verifier.c

26183

memcpy(env->prog->aux->used_maps, env->used_maps,

kernel/bpf/verifier.c

26184

sizeof(env->used_maps[0]) * env->used_map_cnt);

kernel/bpf/verifier.c

26185

env->prog->aux->used_map_cnt = env->used_map_cnt;

kernel/bpf/verifier.c

26187

if (env->used_btf_cnt) {

kernel/bpf/verifier.c

26189

env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0],

kernel/bpf/verifier.c

26190

env->used_btf_cnt,

kernel/bpf/verifier.c

26192

if (!env->prog->aux->used_btfs) {

kernel/bpf/verifier.c

26197

memcpy(env->prog->aux->used_btfs, env->used_btfs,

kernel/bpf/verifier.c

26198

sizeof(env->used_btfs[0]) * env->used_btf_cnt);

kernel/bpf/verifier.c

26199

env->prog->aux->used_btf_cnt = env->used_btf_cnt;

kernel/bpf/verifier.c

26201

if (env->used_map_cnt || env->used_btf_cnt) {

kernel/bpf/verifier.c

26205

convert_pseudo_ld_imm64(env);

kernel/bpf/verifier.c

26208

adjust_btf_func(env);

kernel/bpf/verifier.c

26212

release_insn_arrays(env);

kernel/bpf/verifier.c

26213

if (!env->prog->aux->used_maps)

kernel/bpf/verifier.c

26217

release_maps(env);

kernel/bpf/verifier.c

26218

if (!env->prog->aux->used_btfs)

kernel/bpf/verifier.c

26219

release_btfs(env);

kernel/bpf/verifier.c

26224

if (env->prog->type == BPF_PROG_TYPE_EXT)

kernel/bpf/verifier.c

26225

env->prog->expected_attach_type = 0;

kernel/bpf/verifier.c

26227

*prog = env->prog;

kernel/bpf/verifier.c

26229

module_put(env->attach_btf_mod);

kernel/bpf/verifier.c

26233

clear_insn_aux_data(env, 0, env->prog->len);

kernel/bpf/verifier.c

26234

vfree(env->insn_aux_data);

kernel/bpf/verifier.c

26236

bpf_stack_liveness_free(env);

kernel/bpf/verifier.c

26237

kvfree(env->cfg.insn_postorder);

kernel/bpf/verifier.c

26238

kvfree(env->scc_info);

kernel/bpf/verifier.c

26239

kvfree(env->succ);

kernel/bpf/verifier.c

26240

kvfree(env->gotox_tmp_buf);

kernel/bpf/verifier.c

26241

kvfree(env);

kernel/bpf/verifier.c

2787

static int reg_bounds_sanity_check(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2824

verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "

kernel/bpf/verifier.c

2831

if (env->test_reg_invariants)

kernel/bpf/verifier.c

2881

static void __mark_reg_unknown(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2885

reg->precise = !env->bpf_capable;

kernel/bpf/verifier.c

2888

static void mark_reg_unknown(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2892

verbose(env, "mark_reg_unknown(regs, %u)\n", regno);

kernel/bpf/verifier.c

2895

__mark_reg_not_init(env, regs + regno);

kernel/bpf/verifier.c

2898

__mark_reg_unknown(env, regs + regno);

kernel/bpf/verifier.c

2901

static int __mark_reg_s32_range(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2917

return reg_bounds_sanity_check(env, reg, "s32_range");

kernel/bpf/verifier.c

2920

static void __mark_reg_not_init(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2923

__mark_reg_unknown(env, reg);

kernel/bpf/verifier.c

2927

static void mark_reg_not_init(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2931

verbose(env, "mark_reg_not_init(regs, %u)\n", regno);

kernel/bpf/verifier.c

2934

__mark_reg_not_init(env, regs + regno);

kernel/bpf/verifier.c

2937

__mark_reg_not_init(env, regs + regno);

kernel/bpf/verifier.c

2940

static int mark_btf_ld_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2948

mark_reg_unknown(env, regs, regno);

kernel/bpf/verifier.c

2951

mark_reg_known_zero(env, regs, regno);

kernel/bpf/verifier.c

2956

regs[regno].id = ++env->id_gen;

kernel/bpf/verifier.c

2959

mark_reg_known_zero(env, regs, regno);

kernel/bpf/verifier.c

2964

verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__);

kernel/bpf/verifier.c

2970

static void init_reg_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

2977

mark_reg_not_init(env, regs, i);

kernel/bpf/verifier.c

2983

mark_reg_known_zero(env, regs, BPF_REG_FP);

kernel/bpf/verifier.c

2993

static void init_func_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3001

init_reg_state(env, state);

kernel/bpf/verifier.c

3002

mark_verifier_state_scratched(env);

kernel/bpf/verifier.c

3006

static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3019

elem->next = env->head;

kernel/bpf/verifier.c

3020

elem->log_pos = env->log.end_pos;

kernel/bpf/verifier.c

3021

env->head = elem;

kernel/bpf/verifier.c

3022

env->stack_size++;

kernel/bpf/verifier.c

3023

if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {

kernel/bpf/verifier.c

3024

verbose(env,

kernel/bpf/verifier.c

3026

env->stack_size);

kernel/bpf/verifier.c

3039

init_func_state(env, frame,

kernel/bpf/verifier.c

3061

struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

3063

struct bpf_subprog_info *vals = env->subprog_info;

kernel/bpf/verifier.c

3066

if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)

kernel/bpf/verifier.c

3070

r = env->subprog_cnt - 1;

kernel/bpf/verifier.c

3082

static int find_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

3086

p = bpf_find_containing_subprog(env, off);

kernel/bpf/verifier.c

3089

return p - env->subprog_info;

kernel/bpf/verifier.c

3092

static int add_subprog(struct bpf_verifier_env *env, int off)

kernel/bpf/verifier.c

3094

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

3098

verbose(env, "call to invalid destination\n");

kernel/bpf/verifier.c

3101

ret = find_subprog(env, off);

kernel/bpf/verifier.c

3104

if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {

kernel/bpf/verifier.c

3105

verbose(env, "too many subprograms\n");

kernel/bpf/verifier.c

3109

env->subprog_info[env->subprog_cnt++].start = off;

kernel/bpf/verifier.c

3110

sort(env->subprog_info, env->subprog_cnt,

kernel/bpf/verifier.c

3111

sizeof(env->subprog_info[0]), cmp_subprogs, NULL);

kernel/bpf/verifier.c

3112

return env->subprog_cnt - 1;

kernel/bpf/verifier.c

3115

static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

3117

struct bpf_prog_aux *aux = env->prog->aux;

kernel/bpf/verifier.c

3131

verbose(env, "invalid btf id for main subprog in func_info\n");

kernel/bpf/verifier.c

3142

verbose(env, "multiple exception callback tags for main subprog\n");

kernel/bpf/verifier.c

3148

verbose(env, "exception callback '%s' could not be found in BTF\n", name);

kernel/bpf/verifier.c

3154

verbose(env, "exception callback '%s' must have global linkage\n", name);

kernel/bpf/verifier.c

3166

verbose(env, "invalid exception callback insn_off in func_info: 0\n");

kernel/bpf/verifier.c

3171

verbose(env, "exception callback type id not found in func_info\n");

kernel/bpf/verifier.c

3209

static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc,

kernel/bpf/verifier.c

3256

static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3266

tab = env->prog->aux->kfunc_btf_tab;

kernel/bpf/verifier.c

3271

verbose(env, "too many different module BTFs\n");

kernel/bpf/verifier.c

3275

if (bpfptr_is_null(env->fd_array)) {

kernel/bpf/verifier.c

3276

verbose(env, "kfunc offset > 0 without fd_array is invalid\n");

kernel/bpf/verifier.c

3280

if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,

kernel/bpf/verifier.c

3287

verbose(env, "invalid module BTF fd specified\n");

kernel/bpf/verifier.c

3292

verbose(env, "BTF fd for kfunc is not a module BTF\n");

kernel/bpf/verifier.c

3332

static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)

kernel/bpf/verifier.c

3339

verbose(env, "negative offset disallowed for kernel module function call\n");

kernel/bpf/verifier.c

3343

return __find_kfunc_desc_btf(env, offset);

kernel/bpf/verifier.c

3350

static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3354

char *buf = env->tmp_str_buf;

kernel/bpf/verifier.c

3361

verbose(env, "function name %s%s is too long\n", func_name, KF_IMPL_SUFFIX);

kernel/bpf/verifier.c

3367

verbose(env, "cannot find function %s in BTF\n", buf);

kernel/bpf/verifier.c

3376

static int fetch_kfunc_meta(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3387

verbose(env, "invalid kernel function btf_id %d\n", func_id);

kernel/bpf/verifier.c

3391

btf = find_kfunc_desc_btf(env, offset);

kernel/bpf/verifier.c

3393

verbose(env, "failed to find BTF for kernel function\n");

kernel/bpf/verifier.c

3404

kfunc_flags = btf_kfunc_flags(btf, func_id, env->prog);

kernel/bpf/verifier.c

3408

verbose(env, "kernel btf_id %d is not a function\n", func_id);

kernel/bpf/verifier.c

3419

func_proto = find_kfunc_impl_proto(env, btf, func_name);

kernel/bpf/verifier.c

3424

verbose(env, "kernel function btf_id %d does not have a valid func_proto\n",

kernel/bpf/verifier.c

3439

static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)

kernel/bpf/verifier.c

3450

prog_aux = env->prog->aux;

kernel/bpf/verifier.c

3455

verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");

kernel/bpf/verifier.c

3459

if (!env->prog->jit_requested) {

kernel/bpf/verifier.c

3460

verbose(env, "JIT is required for calling kernel function\n");

kernel/bpf/verifier.c

3465

verbose(env, "JIT does not support calling kernel function\n");

kernel/bpf/verifier.c

3469

if (!env->prog->gpl_compatible) {

kernel/bpf/verifier.c

3470

verbose(env, "cannot call kernel function from non-GPL compatible program\n");

kernel/bpf/verifier.c

3496

if (find_kfunc_desc(env->prog, func_id, offset))

kernel/bpf/verifier.c

3500

verbose(env, "too many different kernel function calls\n");

kernel/bpf/verifier.c

3504

err = fetch_kfunc_meta(env, func_id, offset, &kfunc);

kernel/bpf/verifier.c

3510

verbose(env, "cannot find address for kernel function %s\n", kfunc.name);

kernel/bpf/verifier.c

3515

err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);

kernel/bpf/verifier.c

3520

err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model);

kernel/bpf/verifier.c

3546

static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)

kernel/bpf/verifier.c

3556

verbose(env, "address of kernel func_id %u is out of range\n",

kernel/bpf/verifier.c

3565

static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

3570

tab = env->prog->aux->kfunc_tab;

kernel/bpf/verifier.c

3575

err = set_kfunc_desc_imm(env, &tab->descs[i]);

kernel/bpf/verifier.c

3608

static int add_kfunc_in_insns(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3615

ret = add_kfunc_call(env, insn->imm, insn->off);

kernel/bpf/verifier.c

3623

static int add_subprog_and_kfunc(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

3625

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

3626

int i, ret, insn_cnt = env->prog->len, ex_cb_insn;

kernel/bpf/verifier.c

3627

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

3630

ret = add_subprog(env, 0);

kernel/bpf/verifier.c

3639

if (!env->bpf_capable) {

kernel/bpf/verifier.c

3640

verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");

kernel/bpf/verifier.c

3645

ret = add_subprog(env, i + insn->imm + 1);

kernel/bpf/verifier.c

3647

ret = add_kfunc_call(env, insn->imm, insn->off);

kernel/bpf/verifier.c

3653

ret = bpf_find_exception_callback_insn_off(env);

kernel/bpf/verifier.c

3662

ret = add_subprog(env, ex_cb_insn);

kernel/bpf/verifier.c

3665

for (i = 1; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

3666

if (env->subprog_info[i].start != ex_cb_insn)

kernel/bpf/verifier.c

3668

env->exception_callback_subprog = i;

kernel/bpf/verifier.c

3669

mark_subprog_exc_cb(env, i);

kernel/bpf/verifier.c

3677

subprog[env->subprog_cnt].start = insn_cnt;

kernel/bpf/verifier.c

3679

if (env->log.level & BPF_LOG_LEVEL2)

kernel/bpf/verifier.c

3680

for (i = 0; i < env->subprog_cnt; i++)

kernel/bpf/verifier.c

3681

verbose(env, "func#%d @%d\n", i, subprog[i].start);

kernel/bpf/verifier.c

3686

static int check_subprogs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

3689

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

3690

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

3691

int insn_cnt = env->prog->len;

kernel/bpf/verifier.c

3718

verbose(env, "jump out of range from insn %d to %d\n", i, off);

kernel/bpf/verifier.c

3730

verbose(env, "last insn is not an exit or jmp\n");

kernel/bpf/verifier.c

3735

if (cur_subprog < env->subprog_cnt)

kernel/bpf/verifier.c

374

struct bpf_verifier_env *env = private_data;

kernel/bpf/verifier.c

3742

static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

3748

err = bpf_mark_stack_read(env, reg->frameno, env->insn_idx, BIT(spi - i));

kernel/bpf/verifier.c

3751

mark_stack_slot_scratched(env, spi - i);

kernel/bpf/verifier.c

3756

static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

3766

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

377

if (!bpf_verifier_log_needed(&env->log))

kernel/bpf/verifier.c

3773

return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS);

kernel/bpf/verifier.c

3776

static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

3779

return mark_stack_slot_obj_read(env, reg, spi, nr_slots);

kernel/bpf/verifier.c

3782

static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

3786

spi = irq_flag_get_spi(env, reg);

kernel/bpf/verifier.c

3789

return mark_stack_slot_obj_read(env, reg, spi, 1);

kernel/bpf/verifier.c

381

bpf_verifier_vlog(&env->log, fmt, args);

kernel/bpf/verifier.c

385

static void verbose_invalid_scalar(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

3917

static void mark_insn_zext(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

392

verbose(env, "%s the register %s has", ctx, reg_name);

kernel/bpf/verifier.c

3925

env->insn_aux_data[def_idx - 1].zext_dst = true;

kernel/bpf/verifier.c

3930

static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,

kernel/bpf/verifier.c

3933

struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;

kernel/bpf/verifier.c

3938

verbose(env, "R%d is invalid\n", regno);

kernel/bpf/verifier.c

394

verbose(env, " smin=%lld", reg->smin_value);

kernel/bpf/verifier.c

3942

mark_reg_scratched(env, regno);

kernel/bpf/verifier.c

3949

verbose(env, "R%d !read_ok\n", regno);

kernel/bpf/verifier.c

3957

mark_insn_zext(env, reg);

kernel/bpf/verifier.c

3963

verbose(env, "frame pointer is read only\n");

kernel/bpf/verifier.c

3966

reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;

kernel/bpf/verifier.c

3968

mark_reg_unknown(env, regs, regno);

kernel/bpf/verifier.c

3973

static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

3976

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

3979

return __check_reg_arg(env, state->regs, regno, t);

kernel/bpf/verifier.c

398

verbose(env, " smax=%lld", reg->smax_value);

kernel/bpf/verifier.c

3997

static void mark_jmp_point(struct bpf_verifier_env *env, int idx)

kernel/bpf/verifier.c

3999

env->insn_aux_data[idx].jmp_point = true;

kernel/bpf/verifier.c

4002

static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

4004

return env->insn_aux_data[insn_idx].jmp_point;

kernel/bpf/verifier.c

402

verbose(env, " unknown scalar value");

kernel/bpf/verifier.c

403

verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);

kernel/bpf/verifier.c

4086

static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,

kernel/bpf/verifier.c

4094

if (env->cur_hist_ent) {

kernel/bpf/verifier.c

4098

verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&

kernel/bpf/verifier.c

4099

(env->cur_hist_ent->flags & insn_flags) != insn_flags,

kernel/bpf/verifier.c

4100

env, "insn history: insn_idx %d cur flags %x new flags %x",

kernel/bpf/verifier.c

4101

env->insn_idx, env->cur_hist_ent->flags, insn_flags);

kernel/bpf/verifier.c

4102

env->cur_hist_ent->flags |= insn_flags;

kernel/bpf/verifier.c

4103

verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,

kernel/bpf/verifier.c

4105

env->insn_idx, env->cur_hist_ent->linked_regs);

kernel/bpf/verifier.c

4106

env->cur_hist_ent->linked_regs = linked_regs;

kernel/bpf/verifier.c

4118

p->idx = env->insn_idx;

kernel/bpf/verifier.c

4119

p->prev_idx = env->prev_insn_idx;

kernel/bpf/verifier.c

4123

env->cur_hist_ent = p;

kernel/bpf/verifier.c

4186

static void verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

4191

.private_data = env,

kernel/bpf/verifier.c

4194

print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);

kernel/bpf/verifier.c

4204

struct bpf_verifier_env *env = bt->env;

kernel/bpf/verifier.c

4207

bt->env = env;

kernel/bpf/verifier.c

4224

verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);

kernel/bpf/verifier.c

4234

verifier_bug(bt->env, "subprog exit from frame 0");

kernel/bpf/verifier.c

4390

static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,

kernel/bpf/verifier.c

4393

struct bpf_insn *insn = env->prog->insnsi + idx;

kernel/bpf/verifier.c

440

static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4403

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

4404

fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));

kernel/bpf/verifier.c

4405

verbose(env, "mark_precise: frame%d: regs=%s ",

kernel/bpf/verifier.c

4406

bt->frame, env->tmp_str_buf);

kernel/bpf/verifier.c

4407

bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));

kernel/bpf/verifier.c

4408

verbose(env, "stack=%s before ", env->tmp_str_buf);

kernel/bpf/verifier.c

4409

verbose(env, "%d: ", idx);

kernel/bpf/verifier.c

4410

verbose_insn(env, insn);

kernel/bpf/verifier.c

442

struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;

kernel/bpf/verifier.c

447

static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

451

if (!env->prog->aux->func_info)

kernel/bpf/verifier.c

4513

subprog = find_subprog(env, subprog_insn_idx);

kernel/bpf/verifier.c

4517

if (subprog_is_global(env, subprog)) {

kernel/bpf/verifier.c

4524

verifier_bug_if(idx + 1 != subseq_idx, env,

kernel/bpf/verifier.c

4531

verifier_bug(env, "global subprog unexpected regs %x",

kernel/bpf/verifier.c

454

info = &env->prog->aux->func_info[subprog];

kernel/bpf/verifier.c

4546

verifier_bug(env, "static subprog unexpected regs %x",

kernel/bpf/verifier.c

455

return btf_type_name(env->prog->aux->btf, info->type_id);

kernel/bpf/verifier.c

4554

verifier_bug(env,

kernel/bpf/verifier.c

4579

verifier_bug(env, "callback unexpected regs %x",

kernel/bpf/verifier.c

458

static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

4584

verifier_bug(env, "callback leftover stack slots %llx",

kernel/bpf/verifier.c

460

struct bpf_subprog_info *info = subprog_info(env, subprog);

kernel/bpf/verifier.c

4607

verifier_bug(env, "backtracking call unexpected regs %x",

kernel/bpf/verifier.c

4627

if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))

kernel/bpf/verifier.c

4631

verifier_bug(env, "backtracking exit unexpected regs %x",

kernel/bpf/verifier.c

4645

bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&

kernel/bpf/verifier.c

467

static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)

kernel/bpf/verifier.c

469

return subprog_info(env, subprog)->is_exception_cb;

kernel/bpf/verifier.c

4749

static void mark_all_scalars_precise(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4756

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

4757

verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",

kernel/bpf/verifier.c

4775

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

4776

verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",

kernel/bpf/verifier.c

4787

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

4788

verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",

kernel/bpf/verifier.c

4796

static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)

kernel/bpf/verifier.c

4908

static int __mark_chain_precision(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

4914

struct backtrack_state *bt = &env->bt;

kernel/bpf/verifier.c

4923

if (!env->bpf_capable)

kernel/bpf/verifier.c

4938

verifier_bug(env, "backtracking misuse");

kernel/bpf/verifier.c

4952

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

4953

verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",

kernel/bpf/verifier.c

4980

verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",

kernel/bpf/verifier.c

4991

err = backtrack_insn(env, i, subseq_idx, hist, bt);

kernel/bpf/verifier.c

4994

mark_all_scalars_precise(env, starting_state);

kernel/bpf/verifier.c

5010

if (i >= env->prog->len) {

kernel/bpf/verifier.c

5017

verifier_bug(env, "backtracking idx %d", i);

kernel/bpf/verifier.c

5045

env, "stack slot %d, total slots %d",

kernel/bpf/verifier.c

5061

if (env->log.level & BPF_LOG_LEVEL2) {

kernel/bpf/verifier.c

5062

fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,

kernel/bpf/verifier.c

5064

verbose(env, "mark_precise: frame%d: parent state regs=%s ",

kernel/bpf/verifier.c

5065

fr, env->tmp_str_buf);

kernel/bpf/verifier.c

5066

bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,

kernel/bpf/verifier.c

5068

verbose(env, "stack=%s: ", env->tmp_str_buf);

kernel/bpf/verifier.c

5069

print_verifier_state(env, st, fr, true);

kernel/bpf/verifier.c

5086

mark_all_scalars_precise(env, starting_state);

kernel/bpf/verifier.c

5093

int mark_chain_precision(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

5095

return __mark_chain_precision(env, env->cur_state, regno, NULL);

kernel/bpf/verifier.c

5101

static int mark_chain_precision_batch(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5104

return __mark_chain_precision(env, starting_state, -1, NULL);

kernel/bpf/verifier.c

5162

static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5183

src_reg->id = ++env->id_gen;

kernel/bpf/verifier.c

5192

static void save_register_state(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5206

mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);

kernel/bpf/verifier.c

5220

static void check_fastcall_stack_contract(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5223

struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];

kernel/bpf/verifier.c

5224

struct bpf_insn_aux_data *aux = env->insn_aux_data;

kernel/bpf/verifier.c

5247

static int check_stack_write_fixed_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5255

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

5262

if (!env->allow_ptr_leaks &&

kernel/bpf/verifier.c

5266

verbose(env, "attempt to corrupt spilled pointer on stack\n");

kernel/bpf/verifier.c

5270

cur = env->cur_state->frame[env->cur_state->curframe];

kernel/bpf/verifier.c

5273

if (!env->bypass_spec_v4) {

kernel/bpf/verifier.c

5286

env->insn_aux_data[insn_idx].nospec_result = true;

kernel/bpf/verifier.c

5289

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

5302

bpf_mark_stack_write(env, state->frameno, BIT(spi));

kernel/bpf/verifier.c

5305

check_fastcall_stack_contract(env, state, insn_idx, off);

kernel/bpf/verifier.c

5306

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

5307

if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {

kernel/bpf/verifier.c

5313

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

5314

save_register_state(env, state, spi, reg, size);

kernel/bpf/verifier.c

5319

env->bpf_capable) {

kernel/bpf/verifier.c

5320

struct bpf_reg_state *tmp_reg = &env->fake_reg[0];

kernel/bpf/verifier.c

5325

save_register_state(env, state, spi, tmp_reg, size);

kernel/bpf/verifier.c

5329

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/verifier.c

5330

verbose(env, "invalid size of register spill\n");

kernel/bpf/verifier.c

5334

verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");

kernel/bpf/verifier.c

5337

save_register_state(env, state, spi, reg, size);

kernel/bpf/verifier.c

5357

err = mark_chain_precision(env, value_regno);

kernel/bpf/verifier.c

5370

return push_jmp_history(env, env->cur_state, insn_flags, 0);

kernel/bpf/verifier.c

5393

static int check_stack_write_var_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5403

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

5410

cur = env->cur_state->frame[env->cur_state->curframe];

kernel/bpf/verifier.c

5424

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

5429

check_fastcall_stack_contract(env, state, insn_idx, min_off);

kernel/bpf/verifier.c

5438

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

5440

if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {

kernel/bpf/verifier.c

5452

verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",

kernel/bpf/verifier.c

5487

if (*stype == STACK_INVALID && !env->allow_uninit_stack) {

kernel/bpf/verifier.c

5488

verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",

kernel/bpf/verifier.c

5496

err = mark_chain_precision(env, value_regno);

kernel/bpf/verifier.c

5511

static void mark_reg_stack_read(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5516

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

5525

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

5535

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

5538

mark_reg_unknown(env, state->regs, dst_regno);

kernel/bpf/verifier.c

5551

static int check_stack_read_fixed_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5556

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

5567

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

5568

check_fastcall_stack_contract(env, state, env->insn_idx, off);

kernel/bpf/verifier.c

5569

err = bpf_mark_stack_read(env, reg_state->frameno, env->insn_idx, BIT(spi));

kernel/bpf/verifier.c

558

static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

5581

verbose_linfo(env, env->insn_idx, "; ");

kernel/bpf/verifier.c

5582

verbose(env, "invalid size of register fill\n");

kernel/bpf/verifier.c

5596

if (env->bpf_capable && size == 4 && spill_size == 4 &&

kernel/bpf/verifier.c

5601

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

5625

if (type == STACK_INVALID && env->allow_uninit_stack)

kernel/bpf/verifier.c

5627

verbose(env, "invalid read from stack off %d+%d size %d\n",

kernel/bpf/verifier.c

5634

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

5638

__mark_reg_const_zero(env, &state->regs[dst_regno]);

kernel/bpf/verifier.c

5641

mark_reg_unknown(env, state->regs, dst_regno);

kernel/bpf/verifier.c

5647

if (env->bpf_capable)

kernel/bpf/verifier.c

5651

assign_scalar_id_before_mov(env, reg);

kernel/bpf/verifier.c

5657

} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {

kernel/bpf/verifier.c

5664

verbose(env, "leaking pointer from stack off %d\n",

kernel/bpf/verifier.c

5675

if (type == STACK_INVALID && env->allow_uninit_stack)

kernel/bpf/verifier.c

5677

verbose(env, "invalid read from stack off %d+%d size %d\n",

kernel/bpf/verifier.c

5682

mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);

kernel/bpf/verifier.c

5686

return push_jmp_history(env, env->cur_state, insn_flags, 0);

kernel/bpf/verifier.c

569

verifier_bug(env, "unhandled async callback in is_async_cb_sleepable");

kernel/bpf/verifier.c

5695

static int check_stack_range_initialized(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5701

static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

5703

return cur_regs(env) + regno;

kernel/bpf/verifier.c

5719

static int check_stack_read_var_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5723

struct bpf_reg_state *reg = reg_state(env, ptr_regno);

kernel/bpf/verifier.c

5724

struct bpf_func_state *ptr_state = func(env, reg);

kernel/bpf/verifier.c

5730

err = check_stack_range_initialized(env, ptr_regno, off, size,

kernel/bpf/verifier.c

5737

mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);

kernel/bpf/verifier.c

5738

check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);

kernel/bpf/verifier.c

5751

static int check_stack_read(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5755

struct bpf_reg_state *reg = reg_state(env, ptr_regno);

kernel/bpf/verifier.c

5756

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

5769

verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",

kernel/bpf/verifier.c

578

static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)

kernel/bpf/verifier.c

5784

err = check_stack_read_fixed_off(env, state, off, size,

kernel/bpf/verifier.c

5791

err = check_stack_read_var_off(env, ptr_regno, off, size,

kernel/bpf/verifier.c

580

return is_may_goto_insn(&env->prog->insnsi[insn_idx]);

kernel/bpf/verifier.c

5808

static int check_stack_write(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5812

struct bpf_reg_state *reg = reg_state(env, ptr_regno);

kernel/bpf/verifier.c

5813

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

5818

err = check_stack_write_fixed_off(env, state, off, size,

kernel/bpf/verifier.c

5824

err = check_stack_write_var_off(env, state,

kernel/bpf/verifier.c

5831

static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

5834

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

5839

verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

5845

verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

5854

static int __check_mem_access(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

5864

reg = &cur_regs(env)[regno];

kernel/bpf/verifier.c

5867

verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

5871

verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",

kernel/bpf/verifier.c

5877

verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",

kernel/bpf/verifier.c

5882

verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",

kernel/bpf/verifier.c

5890

static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

5894

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

5913

verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",

kernel/bpf/verifier.c

5917

err = __check_mem_access(env, regno, reg->smin_value + off, size,

kernel/bpf/verifier.c

5920

verbose(env, "R%d min value is outside of the allowed memory range\n",

kernel/bpf/verifier.c

5930

verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",

kernel/bpf/verifier.c

5934

err = __check_mem_access(env, regno, reg->umax_value + off, size,

kernel/bpf/verifier.c

5937

verbose(env, "R%d max value is outside of the allowed memory range\n",

kernel/bpf/verifier.c

5945

static int __check_ptr_off_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5954

verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",

kernel/bpf/verifier.c

5955

reg_type_str(env, reg->type), regno, reg->off);

kernel/bpf/verifier.c

5960

verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",

kernel/bpf/verifier.c

5961

reg_type_str(env, reg->type), regno, reg->off);

kernel/bpf/verifier.c

5969

verbose(env, "variable %s access var_off=%s disallowed\n",

kernel/bpf/verifier.c

5970

reg_type_str(env, reg->type), tn_buf);

kernel/bpf/verifier.c

5977

static int check_ptr_off_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

5980

return __check_ptr_off_reg(env, reg, regno, false);

kernel/bpf/verifier.c

5983

static int map_kptr_match_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6015

if (__check_ptr_off_reg(env, reg, regno, true))

kernel/bpf/verifier.c

6042

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,

kernel/bpf/verifier.c

6048

verbose(env, "invalid kptr access, R%d type=%s%s ", regno,

kernel/bpf/verifier.c

6049

reg_type_str(env, reg->type), reg_name);

kernel/bpf/verifier.c

6050

verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);

kernel/bpf/verifier.c

6052

verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),

kernel/bpf/verifier.c

6055

verbose(env, "\n");

kernel/bpf/verifier.c

6059

static bool in_sleepable(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

6061

return env->cur_state->in_sleepable;

kernel/bpf/verifier.c

6067

static bool in_rcu_cs(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

6069

return env->cur_state->active_rcu_locks ||

kernel/bpf/verifier.c

6070

env->cur_state->active_locks ||

kernel/bpf/verifier.c

6071

!in_sleepable(env);

kernel/bpf/verifier.c

6119

static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)

kernel/bpf/verifier.c

6125

if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {

kernel/bpf/verifier.c

6142

static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

6149

mark_reg_known_zero(env, cur_regs(env), regno);

kernel/bpf/verifier.c

6150

reg = reg_state(env, regno);

kernel/bpf/verifier.c

6153

reg->id = ++env->id_gen;

kernel/bpf/verifier.c

6158

static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

6162

struct bpf_insn *insn = &env->prog->insnsi[insn_idx];

kernel/bpf/verifier.c

6175

verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");

kernel/bpf/verifier.c

6184

verbose(env, "store to referenced kptr disallowed\n");

kernel/bpf/verifier.c

6188

verbose(env, "store to uptr disallowed\n");

kernel/bpf/verifier.c

6194

return mark_uptr_ld_reg(env, value_regno, kptr_field);

kernel/bpf/verifier.c

6199

ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID,

kernel/bpf/verifier.c

6201

btf_ld_kptr_type(env, kptr_field));

kernel/bpf/verifier.c

6205

val_reg = reg_state(env, value_regno);

kernel/bpf/verifier.c

6207

map_kptr_match_type(env, kptr_field, val_reg, value_regno))

kernel/bpf/verifier.c

6211

verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",

kernel/bpf/verifier.c

6216

verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");

kernel/bpf/verifier.c

6235

static int check_map_access(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

6239

struct bpf_verifier_state *vstate = env->cur_state;

kernel/bpf/verifier.c

6247

err = check_mem_region_access(env, regno, off, size, mem_size, zero_size_allowed);

kernel/bpf/verifier.c

6270

verbose(env, "%s cannot be accessed indirectly by helper\n",

kernel/bpf/verifier.c

6275

verbose(env, "%s access cannot have variable offset\n",

kernel/bpf/verifier.c

6280

verbose(env, "%s access misaligned expected=%u off=%llu\n",

kernel/bpf/verifier.c

6286

verbose(env, "%s access size must be BPF_DW\n",

kernel/bpf/verifier.c

6292

verbose(env, "%s cannot be accessed directly by load/store\n",

kernel/bpf/verifier.c

6303

static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6307

enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

kernel/bpf/verifier.c

632

static struct bpf_func_state *func(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6331

env->seen_direct_write = true;

kernel/bpf/verifier.c

6336

env->seen_direct_write = true;

kernel/bpf/verifier.c

6345

static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,

kernel/bpf/verifier.c

6348

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

635

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

6360

verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",

kernel/bpf/verifier.c

6366

__check_mem_access(env, regno, off, size, reg->range,

kernel/bpf/verifier.c

6369

verbose(env, "R%d offset is outside of the packet\n", regno);

kernel/bpf/verifier.c

6379

env->prog->aux->max_pkt_offset =

kernel/bpf/verifier.c

6380

max_t(u32, env->prog->aux->max_pkt_offset,

kernel/bpf/verifier.c

6387

static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,

kernel/bpf/verifier.c

6390

if (env->ops->is_valid_access &&

kernel/bpf/verifier.c

6391

env->ops->is_valid_access(off, size, t, env->prog, info)) {

kernel/bpf/verifier.c

6401

!find_reference_state(env->cur_state, info->ref_obj_id)) {

kernel/bpf/verifier.c

6402

verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",

kernel/bpf/verifier.c

6407

env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;

kernel/bpf/verifier.c

6410

if (env->prog->aux->max_ctx_offset < off + size)

kernel/bpf/verifier.c

6411

env->prog->aux->max_ctx_offset = off + size;

kernel/bpf/verifier.c

6415

verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);

kernel/bpf/verifier.c

6419

static int check_flow_keys_access(struct bpf_verifier_env *env, int off,

kernel/bpf/verifier.c

6424

verbose(env, "invalid access to flow keys off=%d size=%d\n",

kernel/bpf/verifier.c

6431

static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,

kernel/bpf/verifier.c

6435

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6440

verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",

kernel/bpf/verifier.c

6464

env->insn_aux_data[insn_idx].ctx_field_size =

kernel/bpf/verifier.c

6469

verbose(env, "R%d invalid %s access off=%d size=%d\n",

kernel/bpf/verifier.c

6470

regno, reg_type_str(env, reg->type), off, size);

kernel/bpf/verifier.c

6475

static bool is_pointer_value(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6477

return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));

kernel/bpf/verifier.c

6480

static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6482

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6487

static bool is_sk_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6489

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6494

static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6496

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6501

static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6503

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6509

static bool is_arena_reg(struct bpf_verifier_env *env, int regno)

kernel/bpf/verifier.c

6511

const struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

6519

static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

6522

if (is_ctx_reg(env, regno))

kernel/bpf/verifier.c

6524

if (is_pkt_reg(env, regno))

kernel/bpf/verifier.c

6526

if (is_flow_key_reg(env, regno))

kernel/bpf/verifier.c

6528

if (is_sk_reg(env, regno))

kernel/bpf/verifier.c

6530

if (is_arena_reg(env, regno))

kernel/bpf/verifier.c

654

static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

6579

static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

660

verbose(env, "%s has to be at a constant offset\n", obj_kind);

kernel/bpf/verifier.c

6605

verbose(env,

kernel/bpf/verifier.c

6614

static int check_generic_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6630

verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",

kernel/bpf/verifier.c

6638

static int check_ptr_alignment(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6642

bool strict = env->strict_alignment || strict_alignment_once;

kernel/bpf/verifier.c

6651

return check_pkt_ptr_alignment(env, reg, off, size, strict);

kernel/bpf/verifier.c

666

verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);

kernel/bpf/verifier.c

6691

return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,

kernel/bpf/verifier.c

672

verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);

kernel/bpf/verifier.c

6723

static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)

kernel/bpf/verifier.c

6725

if (env->prog->jit_requested)

kernel/bpf/verifier.c

6740

static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,

kernel/bpf/verifier.c

6743

struct bpf_subprog_info *subprog = env->subprog_info;

kernel/bpf/verifier.c

6744

struct bpf_insn *insn = env->prog->insnsi;

kernel/bpf/verifier.c

676

if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))

kernel/bpf/verifier.c

6775

verbose(env,

kernel/bpf/verifier.c

6781

subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

6794

verbose(env, "stack size of subprog %d is %d. Too large\n",

kernel/bpf/verifier.c

6801

verbose(env, "combined stack size of %d calls is %d. Too large\n",

kernel/bpf/verifier.c

681

static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

6826

verbose(env,

kernel/bpf/verifier.c

683

return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);

kernel/bpf/verifier.c

6840

sidx = find_subprog(env, next_insn);

kernel/bpf/verifier.c

6841

if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))

kernel/bpf/verifier.c

6845

verifier_bug(env, "subprog has tail_call and async cb");

kernel/bpf/verifier.c

6852

verbose(env, "insn %d cannot call exception cb directly", i);

kernel/bpf/verifier.c

686

static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)

kernel/bpf/verifier.c

6866

verbose(env, "the call stack of %d frames is too deep !\n",

kernel/bpf/verifier.c

688

return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);

kernel/bpf/verifier.c

6880

verbose(env, "cannot tail call within exception cb\n");

kernel/bpf/verifier.c

6886

env->prog->aux->tail_call_reachable = true;

kernel/bpf/verifier.c

6894

depth -= round_up_stack_depth(env, subprog[idx].stack_depth);

kernel/bpf/verifier.c

6901

static int check_max_stack_depth(struct bpf_verifier_env *env)

kernel/bpf/verifier.c

6904

struct bpf_subprog_info *si = env->subprog_info;

kernel/bpf/verifier.c

6908

for (int i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

691

static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

6916

priv_stack_mode = bpf_enable_priv_stack(env->prog);

kernel/bpf/verifier.c

6926

for (int i = env->subprog_cnt - 1; i >= 0; i--) {

kernel/bpf/verifier.c

6929

ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);

kernel/bpf/verifier.c

693

return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);

kernel/bpf/verifier.c

6935

for (int i = 0; i < env->subprog_cnt; i++) {

kernel/bpf/verifier.c

6937

env->prog->aux->jits_use_priv_stack = true;

kernel/bpf/verifier.c

6946

static int get_callee_stack_depth(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6951

subprog = find_subprog(env, start);

kernel/bpf/verifier.c

6952

if (verifier_bug_if(subprog < 0, env, "get stack depth: no program at insn %d", start))

kernel/bpf/verifier.c

6954

return env->subprog_info[subprog].stack_depth;

kernel/bpf/verifier.c

6958

static int __check_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6964

verbose(env,

kernel/bpf/verifier.c

6973

verbose(env,

kernel/bpf/verifier.c

6982

static int check_tp_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

6988

err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);

kernel/bpf/verifier.c

6992

if (off + size > env->prog->aux->max_tp_access)

kernel/bpf/verifier.c

6993

env->prog->aux->max_tp_access = off + size;

kernel/bpf/verifier.c

6998

static int check_buffer_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7007

err = __check_buffer_access(env, buf_info, reg, regno, off, size);

kernel/bpf/verifier.c

7332

static bool type_is_rcu(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7341

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");

kernel/bpf/verifier.c

7344

static bool type_is_rcu_or_null(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7352

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");

kernel/bpf/verifier.c

7355

static bool type_is_trusted(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7364

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");

kernel/bpf/verifier.c

7367

static bool type_is_trusted_or_null(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7375

return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,

kernel/bpf/verifier.c

7379

static int check_ptr_to_btf_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7393

if (!env->allow_ptr_leaks) {

kernel/bpf/verifier.c

7394

verbose(env,

kernel/bpf/verifier.c

7399

if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {

kernel/bpf/verifier.c

7400

verbose(env,

kernel/bpf/verifier.c

7406

verbose(env,

kernel/bpf/verifier.c

7415

verbose(env,

kernel/bpf/verifier.c

7422

verbose(env,

kernel/bpf/verifier.c

7429

verbose(env,

kernel/bpf/verifier.c

7435

if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {

kernel/bpf/verifier.c

7437

verifier_bug(env, "reg->btf must be kernel btf");

kernel/bpf/verifier.c

7440

ret = env->ops->btf_struct_access(&env->log, reg, off, size);

kernel/bpf/verifier.c

7447

verbose(env, "only read is supported\n");

kernel/bpf/verifier.c

745

static void __mark_reg_not_init(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7453

verifier_bug(env, "ref_obj_id for allocated object must be non-zero");

kernel/bpf/verifier.c

7457

ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);

kernel/bpf/verifier.c

748

static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7485

if (type_is_trusted(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

7487

} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

7489

} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {

kernel/bpf/verifier.c

7490

if (type_is_rcu(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

7494

type_is_rcu_or_null(env, reg, field_name, btf_id)) {

kernel/bpf/verifier.c

7499

if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&

kernel/bpf/verifier.c

7524

ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);

kernel/bpf/verifier.c

753

int id = ++env->id_gen;

kernel/bpf/verifier.c

7532

static int check_ptr_to_map_access(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7548

verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");

kernel/bpf/verifier.c

7553

verbose(env, "map_ptr access not supported for map type %d\n",

kernel/bpf/verifier.c

7561

if (!env->allow_ptr_leaks) {

kernel/bpf/verifier.c

7562

verbose(env,

kernel/bpf/verifier.c

7569

verbose(env, "R%d is %s invalid negative access: off=%d\n",

kernel/bpf/verifier.c

7575

verbose(env, "only read from %s is supported\n", tname);

kernel/bpf/verifier.c

7581

ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID,

kernel/bpf/verifier.c

7585

ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);

kernel/bpf/verifier.c

759

static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7590

ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);

kernel/bpf/verifier.c

7604

static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7611

if (t == BPF_WRITE || env->allow_uninit_stack)

kernel/bpf/verifier.c

7627

struct bpf_verifier_env *env,

kernel/bpf/verifier.c

763

__mark_dynptr_reg(reg, type, true, ++env->id_gen);

kernel/bpf/verifier.c

7631

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

7632

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

7648

verbose(env, "invalid unbounded variable-offset%s stack R%d\n",

kernel/bpf/verifier.c

7656

err = check_stack_slot_within_bounds(env, min_off, state, type);

kernel/bpf/verifier.c

766

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

7667

verbose(env, "invalid%s stack R%d off=%d size=%d\n",

kernel/bpf/verifier.c

7673

verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",

kernel/bpf/verifier.c

7682

return grow_stack_state(env, state, -min_off /* size */);

kernel/bpf/verifier.c

769

static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

7702

static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,

kernel/bpf/verifier.c

7706

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

7715

err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);

kernel/bpf/verifier.c

772

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

7724

verbose(env, "write to change key R%d not allowed\n", regno);

kernel/bpf/verifier.c

7728

err = check_mem_region_access(env, regno, off, size,

kernel/bpf/verifier.c

7733

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7738

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

7739

verbose(env, "R%d leaks addr into map\n", value_regno);

kernel/bpf/verifier.c

7742

err = check_map_access_type(env, regno, off, size, t);

kernel/bpf/verifier.c

7745

err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);

kernel/bpf/verifier.c

7752

err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);

kernel/bpf/verifier.c

776

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

7776

verbose(env, "Invalid read of %d bytes from insn_array\n",

kernel/bpf/verifier.c

7783

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7791

verbose(env, "R%d invalid mem access '%s'\n", regno,

kernel/bpf/verifier.c

7792

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7797

verbose(env, "R%d cannot write into %s\n",

kernel/bpf/verifier.c

7798

regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7803

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

7804

verbose(env, "R%d leaks addr into mem\n", value_regno);

kernel/bpf/verifier.c

7813

err = check_mem_region_access(env, regno, off, size,

kernel/bpf/verifier.c

7816

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7822

.log = &env->log,

kernel/bpf/verifier.c

7826

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

7827

verbose(env, "R%d leaks addr into ctx\n", value_regno);

kernel/bpf/verifier.c

7831

err = check_ptr_off_reg(env, reg, regno);

kernel/bpf/verifier.c

7835

err = check_ctx_access(env, insn_idx, off, size, t, &info);

kernel/bpf/verifier.c

7837

verbose_linfo(env, insn_idx, "; ");

kernel/bpf/verifier.c

7844

if (info.is_retval && get_func_retval_range(env->prog, &range)) {

kernel/bpf/verifier.c

7845

err = __mark_reg_s32_range(env, regs, value_regno,

kernel/bpf/verifier.c

7850

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7853

mark_reg_known_zero(env, regs,

kernel/bpf/verifier.c

7856

regs[value_regno].id = ++env->id_gen;

kernel/bpf/verifier.c

7874

err = check_stack_access_within_bounds(env, regno, off, size, t);

kernel/bpf/verifier.c

7879

err = check_stack_read(env, regno, off, size,

kernel/bpf/verifier.c

7882

err = check_stack_write(env, regno, off, size,

kernel/bpf/verifier.c

7885

if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {

kernel/bpf/verifier.c

7886

verbose(env, "cannot write into packet\n");

kernel/bpf/verifier.c

789

err = destroy_if_dynptr_stack_slot(env, state, spi);

kernel/bpf/verifier.c

7890

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

7891

verbose(env, "R%d leaks addr into packet\n",

kernel/bpf/verifier.c

7895

err = check_packet_access(env, regno, off, size, false);

kernel/bpf/verifier.c

7897

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7900

is_pointer_value(env, value_regno)) {

kernel/bpf/verifier.c

7901

verbose(env, "R%d leaks addr into flow keys\n",

kernel/bpf/verifier.c

7906

err = check_flow_keys_access(env, off, size);

kernel/bpf/verifier.c

7908

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7911

verbose(env, "R%d cannot write into %s\n",

kernel/bpf/verifier.c

7912

regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7915

err = check_sock_access(env, insn_idx, regno, off, size, t);

kernel/bpf/verifier.c

7917

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7919

err = check_tp_buffer_access(env, reg, regno, off, size);

kernel/bpf/verifier.c

792

err = destroy_if_dynptr_stack_slot(env, state, spi - 1);

kernel/bpf/verifier.c

7921

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7924

err = check_ptr_to_btf_access(env, regs, regno, off, size, t,

kernel/bpf/verifier.c

7927

err = check_ptr_to_map_access(env, regs, regno, off, size, t,

kernel/bpf/verifier.c

7936

verbose(env, "R%d cannot write into %s\n",

kernel/bpf/verifier.c

7937

regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7940

max_access = &env->prog->aux->max_rdonly_access;

kernel/bpf/verifier.c

7942

max_access = &env->prog->aux->max_rdwr_access;

kernel/bpf/verifier.c

7945

err = check_buffer_access(env, reg, regno, off, size, false,

kernel/bpf/verifier.c

7949

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7952

mark_reg_unknown(env, regs, value_regno);

kernel/bpf/verifier.c

7954

verbose(env, "R%d invalid mem access '%s'\n", regno,

kernel/bpf/verifier.c

7955

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

7970

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,

kernel/bpf/verifier.c

7973

static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

7977

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

7982

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

7987

err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);

kernel/bpf/verifier.c

7996

err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,

kernel/bpf/verifier.c

7999

err = err ?: save_aux_ptr_type(env, src_reg_type,

kernel/bpf/verifier.c

8001

err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);

kernel/bpf/verifier.c

8006

static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,

kernel/bpf/verifier.c

8009

struct bpf_reg_state *regs = cur_regs(env);

kernel/bpf/verifier.c

8014

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

8019

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

8026

err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,

kernel/bpf/verifier.c

8029

err = err ?: save_aux_ptr_type(env, dst_reg_type, false);

kernel/bpf/verifier.c

8034

static int check_atomic_rmw(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8041

verbose(env, "invalid atomic operand size\n");

kernel/bpf/verifier.c

8046

err = check_reg_arg(env, insn->src_reg, SRC_OP);

kernel/bpf/verifier.c

805

mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,

kernel/bpf/verifier.c

8051

err = check_reg_arg(env, insn->dst_reg, SRC_OP);

kernel/bpf/verifier.c

8059

err = check_reg_arg(env, aux_reg, SRC_OP);

kernel/bpf/verifier.c

8063

if (is_pointer_value(env, aux_reg)) {

kernel/bpf/verifier.c

8064

verbose(env, "R%d leaks addr into mem\n", aux_reg);

kernel/bpf/verifier.c

8069

if (is_pointer_value(env, insn->src_reg)) {

kernel/bpf/verifier.c

8070

verbose(env, "R%d leaks addr into mem\n", insn->src_reg);

kernel/bpf/verifier.c

8074

if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {

kernel/bpf/verifier.c

8075

verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",

kernel/bpf/verifier.c

8077

reg_type_str(env, reg_state(env, insn->dst_reg)->type));

kernel/bpf/verifier.c

8088

err = check_reg_arg(env, load_reg, DST_OP);

kernel/bpf/verifier.c

8101

err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,

kernel/bpf/verifier.c

8104

err = check_mem_access(env, env->insn_idx, insn->dst_reg,

kernel/bpf/verifier.c

8110

if (is_arena_reg(env, insn->dst_reg)) {

kernel/bpf/verifier.c

8111

err = save_aux_ptr_type(env, PTR_TO_ARENA, false);

kernel/bpf/verifier.c

8116

err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,

kernel/bpf/verifier.c

8123

static int check_atomic_load(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8128

err = check_load_mem(env, insn, true, false, false, "atomic_load");

kernel/bpf/verifier.c

8132

if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {

kernel/bpf/verifier.c

8133

verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",

kernel/bpf/verifier.c

8135

reg_type_str(env, reg_state(env, insn->src_reg)->type));

kernel/bpf/verifier.c

8142

static int check_atomic_store(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8147

err = check_store_reg(env, insn, true);

kernel/bpf/verifier.c

815

id = acquire_reference(env, insn_idx);

kernel/bpf/verifier.c

8151

if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {

kernel/bpf/verifier.c

8152

verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",

kernel/bpf/verifier.c

8154

reg_type_str(env, reg_state(env, insn->dst_reg)->type));

kernel/bpf/verifier.c

8161

static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)

kernel/bpf/verifier.c

8174

return check_atomic_rmw(env, insn);

kernel/bpf/verifier.c

8177

verbose(env,

kernel/bpf/verifier.c

8181

return check_atomic_load(env, insn);

kernel/bpf/verifier.c

8184

verbose(env,

kernel/bpf/verifier.c

8188

return check_atomic_store(env, insn);

kernel/bpf/verifier.c

8190

verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",

kernel/bpf/verifier.c

8207

struct bpf_verifier_env *env, int regno, int off,

kernel/bpf/verifier.c

8211

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8212

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

8220

verbose(env, "invalid zero-sized read\n");

kernel/bpf/verifier.c

8227

err = check_stack_access_within_bounds(env, regno, off, access_size, type);

kernel/bpf/verifier.c

824

bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));

kernel/bpf/verifier.c

8240

if (!env->bypass_spec_v1) {

kernel/bpf/verifier.c

8244

verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",

kernel/bpf/verifier.c

8283

verbose(env, "potential write to dynptr at off=%d disallowed\n", i);

kernel/bpf/verifier.c

829

static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)

kernel/bpf/verifier.c

8298

verbose(env, "allocated_stack too small\n");

kernel/bpf/verifier.c

8306

(*stype == STACK_INVALID && env->allow_uninit_stack)) {

kernel/bpf/verifier.c

8316

env->allow_ptr_leaks)) {

kernel/bpf/verifier.c

8318

__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);

kernel/bpf/verifier.c

8326

verbose(env, "invalid read from stack R%d off %d+%d size %d\n",

kernel/bpf/verifier.c

8332

verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n",

kernel/bpf/verifier.c

8340

err = bpf_mark_stack_read(env, reg->frameno, env->insn_idx, BIT(spi));

kernel/bpf/verifier.c

8352

static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

8357

struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];

kernel/bpf/verifier.c

8363

return check_packet_access(env, regno, reg->off, access_size,

kernel/bpf/verifier.c

8367

verbose(env, "R%d cannot write into %s\n", regno,

kernel/bpf/verifier.c

8368

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

8371

return check_mem_region_access(env, regno, reg->off, access_size,

kernel/bpf/verifier.c

8374

if (check_map_access_type(env, regno, reg->off, access_size, access_type))

kernel/bpf/verifier.c

8376

return check_map_access(env, regno, reg->off, access_size,

kernel/bpf/verifier.c

838

__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);

kernel/bpf/verifier.c

8381

verbose(env, "R%d cannot write into %s\n", regno,

kernel/bpf/verifier.c

8382

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

8386

return check_mem_region_access(env, regno, reg->off,

kernel/bpf/verifier.c

839

__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);

kernel/bpf/verifier.c

8392

verbose(env, "R%d cannot write into %s\n", regno,

kernel/bpf/verifier.c

8393

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

8397

max_access = &env->prog->aux->max_rdonly_access;

kernel/bpf/verifier.c

8399

max_access = &env->prog->aux->max_rdwr_access;

kernel/bpf/verifier.c

8401

return check_buffer_access(env, reg, regno, reg->off,

kernel/bpf/verifier.c

8406

env,

kernel/bpf/verifier.c

841

bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));

kernel/bpf/verifier.c

8410

return check_ptr_to_btf_access(env, regs, regno, reg->off,

kernel/bpf/verifier.c

8418

if (!env->ops->convert_ctx_access) {

kernel/bpf/verifier.c

8425

return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,

kernel/bpf/verifier.c

8436

verbose(env, "R%d type=%s ", regno,

kernel/bpf/verifier.c

8437

reg_type_str(env, reg->type));

kernel/bpf/verifier.c

8438

verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));

kernel/bpf/verifier.c

844

static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

8449

static int check_mem_size_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

846

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

8476

verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",

kernel/bpf/verifier.c

8482

verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",

kernel/bpf/verifier.c

8488

verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",

kernel/bpf/verifier.c

8492

err = check_helper_mem_access(env, regno - 1, reg->umax_value,

kernel/bpf/verifier.c

8495

err = mark_chain_precision(env, regno);

kernel/bpf/verifier.c

8499

static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

8518

err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL);

kernel/bpf/verifier.c

8519

err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL);

kernel/bpf/verifier.c

8527

static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,

kernel/bpf/verifier.c

8530

struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];

kernel/bpf/verifier.c

8545

err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta);

kernel/bpf/verifier.c

8546

err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta);

kernel/bpf/verifier.c

855

verifier_bug(env, "CONST_PTR_TO_DYNPTR cannot be released");

kernel/bpf/verifier.c

858

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

8582

static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)

kernel/bpf/verifier.c

8586

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8587

struct bpf_verifier_state *cur = env->cur_state;

kernel/bpf/verifier.c

8598

verbose(env,

kernel/bpf/verifier.c

8606

verbose(env,

kernel/bpf/verifier.c

8617

verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local",

kernel/bpf/verifier.c

8623

verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n",

kernel/bpf/verifier.c

863

invalidate_dynptr(env, state, spi);

kernel/bpf/verifier.c

8637

if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) {

kernel/bpf/verifier.c

8638

verbose(env,

kernel/bpf/verifier.c

8643

if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) {

kernel/bpf/verifier.c

8644

verbose(env, "Acquiring the same lock again, AA deadlock detected\n");

kernel/bpf/verifier.c

8655

err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr);

kernel/bpf/verifier.c

8657

verbose(env, "Failed to acquire lock state\n");

kernel/bpf/verifier.c

8670

verbose(env, "%s_unlock without taking a lock\n", lock_str);

kernel/bpf/verifier.c

8681

verbose(env, "%s_unlock of different lock\n", lock_str);

kernel/bpf/verifier.c

8685

verbose(env, "%s_unlock cannot be out of order\n", lock_str);

kernel/bpf/verifier.c

8689

verbose(env, "%s_unlock of different lock\n", lock_str);

kernel/bpf/verifier.c

8693

invalidate_non_owning_refs(env);

kernel/bpf/verifier.c

8699

static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

8703

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8711

verbose(env,

kernel/bpf/verifier.c

8717

verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name,

kernel/bpf/verifier.c

8722

verbose(env, "map '%s' has no valid %s\n", map->name, struct_name);

kernel/bpf/verifier.c

8736

verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);

kernel/bpf/verifier.c

8740

verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n",

kernel/bpf/verifier.c

8745

verifier_bug(env, "Two map pointers in a %s helper", struct_name);

kernel/bpf/verifier.c

8753

static int process_timer_func(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

8757

verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n");

kernel/bpf/verifier.c

8760

return check_map_field_pointer(env, regno, BPF_TIMER, map);

kernel/bpf/verifier.c

8763

static int process_timer_helper(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

8766

return process_timer_func(env, regno, &meta->map);

kernel/bpf/verifier.c

8769

static int process_timer_kfunc(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

877

WARN_ON_ONCE(release_reference(env, ref_obj_id));

kernel/bpf/verifier.c

8772

return process_timer_func(env, regno, &meta->map);

kernel/bpf/verifier.c

8775

static int process_kptr_func(struct bpf_verifier_env *env, int regno,

kernel/bpf/verifier.c

8778

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8789

verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",

kernel/bpf/verifier.c

8798

verbose(env,

kernel/bpf/verifier.c

8805

verbose(env, "R%d has no valid kptr\n", regno);

kernel/bpf/verifier.c

8812

verbose(env, "off=%d doesn't point to kptr\n", kptr_off);

kernel/bpf/verifier.c

8816

verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);

kernel/bpf/verifier.c

8848

static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,

kernel/bpf/verifier.c

8851

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8855

verbose(env,

kernel/bpf/verifier.c

8865

verifier_bug(env, "misconfigured dynptr helper type flags");

kernel/bpf/verifier.c

8887

if (!is_dynptr_reg_valid_uninit(env, reg)) {

kernel/bpf/verifier.c

8888

verbose(env, "Dynptr has to be an uninitialized dynptr\n");

kernel/bpf/verifier.c

889

verifier_bug(env, "misconfigured ref_obj_id");

kernel/bpf/verifier.c

8894

err = check_mem_access(env, insn_idx, regno,

kernel/bpf/verifier.c

8900

err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);

kernel/bpf/verifier.c

8904

verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");

kernel/bpf/verifier.c

8908

if (!is_dynptr_reg_valid_init(env, reg)) {

kernel/bpf/verifier.c

8909

verbose(env,

kernel/bpf/verifier.c

8916

if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {

kernel/bpf/verifier.c

8917

verbose(env,

kernel/bpf/verifier.c

8923

err = mark_dynptr_read(env, reg);

kernel/bpf/verifier.c

8928

static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)

kernel/bpf/verifier.c

893

invalidate_dynptr(env, state, i);

kernel/bpf/verifier.c

8930

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

8968

static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,

kernel/bpf/verifier.c

8971

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

8976

verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1);

kernel/bpf/verifier.c

8988

verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1);

kernel/bpf/verifier.c

899

static void __mark_reg_unknown(const struct bpf_verifier_env *env,

kernel/bpf/verifier.c

8996

if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {

kernel/bpf/verifier.c

8997

verbose(env, "expected uninitialized iter_%s as arg #%d\n",

kernel/bpf/verifier.c

9003

err = check_mem_access(env, insn_idx, regno,

kernel/bpf/verifier.c

9009

err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);

kernel/bpf/verifier.c

9016

err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);

kernel/bpf/verifier.c

902

static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

9021

verbose(env, "expected an initialized iter_%s as arg #%d\n",

kernel/bpf/verifier.c

9025

verbose(env, "expected an RCU CS when using %s\n", meta->func_name);

kernel/bpf/verifier.c

9031

spi = iter_get_spi(env, reg, nr_slots);

kernel/bpf/verifier.c

9035

err = mark_iter_read(env, reg, spi, nr_slots);

kernel/bpf/verifier.c

904

if (!env->allow_ptr_leaks)

kernel/bpf/verifier.c

9042

meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);

kernel/bpf/verifier.c

9045

err = unmark_stack_slots_iter(env, reg, nr_slots);

kernel/bpf/verifier.c

905

__mark_reg_not_init(env, reg);

kernel/bpf/verifier.c

9057

static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9066

head = explored_state(env, insn_idx);

kernel/bpf/verifier.c

907

__mark_reg_unknown(env, reg);

kernel/bpf/verifier.c

9081

static void reset_idmap_scratch(struct bpf_verifier_env *env);

kernel/bpf/verifier.c

9096

static void maybe_widen_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

910

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9105

__mark_reg_unknown(env, rcur);

kernel/bpf/verifier.c

9108

static int widen_imprecise_scalars(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9120

maybe_widen_reg(env,

kernel/bpf/verifier.c

9131

maybe_widen_reg(env,

kernel/bpf/verifier.c

9226

static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,

kernel/bpf/verifier.c

9229

struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;

kernel/bpf/verifier.c

9239

verifier_bug(env, "unexpected iterator state %d (%s)",

kernel/bpf/verifier.c

9250

verifier_bug(env, "bad parent state for iter next call");

kernel/bpf/verifier.c

9257

prev_st = find_prev_entry(env, cur_st->parent, insn_idx);

kernel/bpf/verifier.c

9259

queued_st = push_stack(env, insn_idx + 1, insn_idx, false);

kernel/bpf/verifier.c

9267

widen_imprecise_scalars(env, prev_st, queued_st);

kernel/bpf/verifier.c

9276

__mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);

kernel/bpf/verifier.c

930

verbose(env, "cannot overwrite referenced dynptr\n");

kernel/bpf/verifier.c

9303

static int resolve_map_arg_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9309

verifier_bug(env, "invalid map_ptr to access map->type");

kernel/bpf/verifier.c

9319

verbose(env, "invalid arg_type for sockmap/sockhash\n");

kernel/bpf/verifier.c

934

mark_stack_slot_scratched(env, spi);

kernel/bpf/verifier.c

935

mark_stack_slot_scratched(env, spi - 1);

kernel/bpf/verifier.c

9444

static int check_reg_type(struct bpf_verifier_env *env, u32 regno,

kernel/bpf/verifier.c

9449

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

945

bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({

kernel/bpf/verifier.c

9456

verifier_bug(env, "unsupported arg type %d", arg_type);

kernel/bpf/verifier.c

9494

verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

9496

verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));

kernel/bpf/verifier.c

9497

verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));

kernel/bpf/verifier.c

950

mark_reg_invalid(env, dreg);

kernel/bpf/verifier.c

9506

verbose(env,

kernel/bpf/verifier.c

9509

regno, reg_type_str(env, reg->type));

kernel/bpf/verifier.c

9532

verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);

kernel/bpf/verifier.c

9538

verifier_bug(env, "missing arg compatible BTF ID");

kernel/bpf/verifier.c

9545

if (map_kptr_match_type(env, meta->kptr_field, reg, regno))

kernel/bpf/verifier.c

9549

verbose(env, "verifier internal error:");

kernel/bpf/verifier.c

9550

verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",

kernel/bpf/verifier.c

9555

if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,

kernel/bpf/verifier.c

9558

verbose(env, "R%d is of type %s but %s is expected\n",

kernel/bpf/verifier.c

956

__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);

kernel/bpf/verifier.c

957

__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);

kernel/bpf/verifier.c

9570

verifier_bug(env, "unimplemented handling of MEM_ALLOC");

kernel/bpf/verifier.c

9575

if (map_kptr_match_type(env, meta->kptr_field, reg, regno))

kernel/bpf/verifier.c

9585

verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match");

kernel/bpf/verifier.c

959

bpf_mark_stack_write(env, state->frameno, BIT(spi - 1) | BIT(spi));

kernel/bpf/verifier.c

9608

static int check_func_arg_reg_off(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9635

verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",

kernel/bpf/verifier.c

9639

return __check_ptr_off_reg(env, reg, regno, false);

kernel/bpf/verifier.c

964

static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

9673

return __check_ptr_off_reg(env, reg, regno, true);

kernel/bpf/verifier.c

9675

return __check_ptr_off_reg(env, reg, regno, false);

kernel/bpf/verifier.c

9679

static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9689

verbose(env, "verifier internal error: multiple dynptr args\n");

kernel/bpf/verifier.c

9696

verbose(env, "verifier internal error: no dynptr arg found\n");

kernel/bpf/verifier.c

9701

static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

9703

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

9708

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

971

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

9714

static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

9716

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

9721

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

9727

static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9730

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

9738

verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");

kernel/bpf/verifier.c

9745

static int check_reg_const_str(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9758

verbose(env, "R%d points to insn_array map which cannot be used as const string\n", regno);

kernel/bpf/verifier.c

9763

verbose(env, "R%d does not point to a readonly map'\n", regno);

kernel/bpf/verifier.c

9768

verbose(env, "R%d is not a constant address'\n", regno);

kernel/bpf/verifier.c

9773

verbose(env, "no direct value access support for this map type\n");

kernel/bpf/verifier.c

9777

err = check_map_access(env, regno, reg->off,

kernel/bpf/verifier.c

9786

verbose(env, "direct value access on string failed\n");

kernel/bpf/verifier.c

9792

verbose(env, "string is not zero-terminated\n");

kernel/bpf/verifier.c

9799

static int get_constant_map_key(struct bpf_verifier_env *env,

kernel/bpf/verifier.c

9804

struct bpf_func_state *state = func(env, key);

kernel/bpf/verifier.c

9813

if (!env->bpf_capable)

kernel/bpf/verifier.c

9850

bt_set_frame_slot(&env->bt, key->frameno, spi);

kernel/bpf/verifier.c

9851

err = mark_chain_precision_batch(env, env->cur_state);

kernel/bpf/verifier.c

9861

static int check_func_arg(struct bpf_verifier_env *env, u32 arg,

kernel/bpf/verifier.c

9867

struct bpf_reg_state *reg = reg_state(env, regno);

kernel/bpf/verifier.c

9877

err = check_reg_arg(env, regno, SRC_OP);

kernel/bpf/verifier.c

9882

if (is_pointer_value(env, regno)) {

kernel/bpf/verifier.c

9883

verbose(env, "R%d leaks addr into helper function\n",

kernel/bpf/verifier.c

9891

!may_access_direct_pkt_data(env, meta, BPF_READ)) {

kernel/bpf/verifier.c

9892

verbose(env, "helper access to the packet is not allowed\n");

kernel/bpf/verifier.c

9897

err = resolve_map_arg_type(env, meta, &arg_type);

kernel/bpf/verifier.c

9913

err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);

kernel/bpf/verifier.c

9917

err = check_func_arg_reg_off(env, reg, regno, arg_type);

kernel/bpf/verifier.c

992

static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)

kernel/bpf/verifier.c

9924

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

9933

spi = dynptr_get_spi(env, reg);

kernel/bpf/verifier.c

9935

verbose(env, "arg %d is an unacquired reference\n", regno);

kernel/bpf/verifier.c

9939

verbose(env, "cannot release unowned const bpf_dynptr\n");

kernel/bpf/verifier.c

994

struct bpf_func_state *state = func(env, reg);

kernel/bpf/verifier.c

9943

verbose(env, "R%d must be referenced when passed to release function\n",

kernel/bpf/verifier.c

9948

verifier_bug(env, "more than one release argument");

kernel/bpf/verifier.c

9956

verbose(env, "more than one arg with ref_obj_id R%d %u %u",

kernel/bpf/verifier.c

9982

verbose(env,

kernel/sched/ext.c

3831

static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)

kernel/sched/ext.c

3835

return add_uevent_var(env, "SCXOPS=%s", sch->ops.name);

kernel/sched/fair.c

10363

sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group)

kernel/sched/fair.c

10373

return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu));

kernel/sched/fair.c

10387

static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs,

kernel/sched/fair.c

10390

if (!env->idle)

kernel/sched/fair.c

10406

static inline long sibling_imbalance(struct lb_env *env,

kernel/sched/fair.c

10414

if (!env->idle || !busiest->sum_nr_running)

kernel/sched/fair.c

10463

static inline void update_sg_lb_stats(struct lb_env *env,

kernel/sched/fair.c

10470

int i, nr_running, local_group, sd_flags = env->sd->flags;

kernel/sched/fair.c

10471

bool balancing_at_rd = !env->sd->parent;

kernel/sched/fair.c

10477

for_each_cpu_and(i, sched_group_span(group), env->cpus) {

kernel/sched/fair.c

10521

} else if (env->idle && sched_reduced_capacity(rq, env->sd)) {

kernel/sched/fair.c

10533

if (!local_group && env->idle && sgs->sum_h_nr_running &&

kernel/sched/fair.c

10534

sched_group_asym(env, sgs, group))

kernel/sched/fair.c

10538

if (!local_group && smt_balance(env, sgs, group))

kernel/sched/fair.c

10541

sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);

kernel/sched/fair.c

10562

static bool update_sd_pick_busiest(struct lb_env *env,

kernel/sched/fair.c

10579

if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&

kernel/sched/fair.c

10581

(!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||

kernel/sched/fair.c

10693

if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&

kernel/sched/fair.c

10695

(capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))

kernel/sched/fair.c

11062

static void update_idle_cpu_scan(struct lb_env *env,

kernel/sched/fair.c

11076

if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)

kernel/sched/fair.c

11079

llc_weight = per_cpu(sd_llc_size, env->dst_cpu);

kernel/sched/fair.c

11080

if (env->sd->span_weight != llc_weight)

kernel/sched/fair.c

11083

sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, env->dst_cpu));

kernel/sched/fair.c

11120

pct = env->sd->imbalance_pct;

kernel/sched/fair.c

11139

static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)

kernel/sched/fair.c

11141

struct sched_group *sg = env->sd->groups;

kernel/sched/fair.c

11151

local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));

kernel/sched/fair.c

11156

if (env->idle != CPU_NEWLY_IDLE ||

kernel/sched/fair.c

11158

update_group_capacity(env->sd, env->dst_cpu);

kernel/sched/fair.c

11161

update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized);

kernel/sched/fair.c

11163

if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {

kernel/sched/fair.c

11174

} while (sg != env->sd->groups);

kernel/sched/fair.c

11185

if (env->sd->flags & SD_NUMA)

kernel/sched/fair.c

11186

env->fbq_type = fbq_classify_group(&sds->busiest_stat);

kernel/sched/fair.c

11188

if (!env->sd->parent) {

kernel/sched/fair.c

11190

set_rd_overloaded(env->dst_rq->rd, sg_overloaded);

kernel/sched/fair.c

11193

set_rd_overutilized(env->dst_rq->rd, sg_overutilized);

kernel/sched/fair.c

11195

set_rd_overutilized(env->dst_rq->rd, sg_overutilized);

kernel/sched/fair.c

11198

update_idle_cpu_scan(env, sum_util);

kernel/sched/fair.c

11207

static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)

kernel/sched/fair.c

11215

if (env->sd->flags & SD_ASYM_CPUCAPACITY) {

kernel/sched/fair.c

11217

env->migration_type = migrate_misfit;

kernel/sched/fair.c

11218

env->imbalance = 1;

kernel/sched/fair.c

11224

env->migration_type = migrate_load;

kernel/sched/fair.c

11225

env->imbalance = busiest->group_misfit_task_load;

kernel/sched/fair.c

11235

env->migration_type = migrate_task;

kernel/sched/fair.c

11236

env->imbalance = busiest->sum_h_nr_running;

kernel/sched/fair.c

11242

env->migration_type = migrate_task;

kernel/sched/fair.c

11243

env->imbalance = 1;

kernel/sched/fair.c

11254

env->migration_type = migrate_task;

kernel/sched/fair.c

11255

env->imbalance = 1;

kernel/sched/fair.c

11265

!(env->sd->flags & SD_SHARE_LLC)) {

kernel/sched/fair.c

11274

env->migration_type = migrate_util;

kernel/sched/fair.c

11275

env->imbalance = max(local->group_capacity, local->group_util) -

kernel/sched/fair.c

11285

if (env->idle && env->imbalance == 0) {

kernel/sched/fair.c

11286

env->migration_type = migrate_task;

kernel/sched/fair.c

11287

env->imbalance = 1;

kernel/sched/fair.c

11298

env->migration_type = migrate_task;

kernel/sched/fair.c

11299

env->imbalance = sibling_imbalance(env, sds, busiest, local);

kernel/sched/fair.c

11306

env->migration_type = migrate_task;

kernel/sched/fair.c

11307

env->imbalance = max_t(long, 0,

kernel/sched/fair.c

11313

if (env->sd->flags & SD_NUMA) {

kernel/sched/fair.c

11314

env->imbalance = adjust_numa_imbalance(env->imbalance,

kernel/sched/fair.c

11316

env->sd->imb_numa_nr);

kernel/sched/fair.c

11321

env->imbalance >>= 1;

kernel/sched/fair.c

11344

env->imbalance = 0;

kernel/sched/fair.c

11356

env->imbalance = 0;

kernel/sched/fair.c

11370

env->migration_type = migrate_load;

kernel/sched/fair.c

11371

env->imbalance = min(

kernel/sched/fair.c

11409

static struct sched_group *sched_balance_find_src_group(struct lb_env *env)

kernel/sched/fair.c

11420

update_sd_lb_stats(env, &sds);

kernel/sched/fair.c

11432

if (!is_rd_overutilized(env->dst_rq->rd) &&

kernel/sched/fair.c

11433

rcu_dereference_all(env->dst_rq->rd->pd))

kernel/sched/fair.c

11484

env->sd->imbalance_pct * local->avg_load)

kernel/sched/fair.c

11493

sibling_imbalance(env, &sds, busiest, local) > 1)

kernel/sched/fair.c

11497

if (!env->idle) {

kernel/sched/fair.c

11536

calculate_imbalance(env, &sds);

kernel/sched/fair.c

11537

return env->imbalance ? sds.busiest : NULL;

kernel/sched/fair.c

11540

env->imbalance = 0;

kernel/sched/fair.c

11547

static struct rq *sched_balance_find_src_rq(struct lb_env *env,

kernel/sched/fair.c

11555

for_each_cpu_and(i, sched_group_span(group), env->cpus) {

kernel/sched/fair.c

11582

if (rt > env->fbq_type)

kernel/sched/fair.c

11597

if (env->sd->flags & SD_ASYM_CPUCAPACITY &&

kernel/sched/fair.c

11598

!capacity_greater(capacity_of(env->dst_cpu), capacity) &&

kernel/sched/fair.c

11609

if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1)

kernel/sched/fair.c

11612

switch (env->migration_type) {

kernel/sched/fair.c

11620

if (nr_running == 1 && load > env->imbalance &&

kernel/sched/fair.c

11621

!check_cpu_capacity(rq, env->sd))

kernel/sched/fair.c

11693

asym_active_balance(struct lb_env *env)

kernel/sched/fair.c

11705

return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) &&

kernel/sched/fair.c

11706

(sched_asym_prefer(env->dst_cpu, env->src_cpu) ||

kernel/sched/fair.c

11707

!sched_use_asym_prio(env->sd, env->src_cpu));

kernel/sched/fair.c

11711

imbalanced_active_balance(struct lb_env *env)

kernel/sched/fair.c

11713

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

11720

if ((env->migration_type == migrate_task) &&

kernel/sched/fair.c

11727

static int need_active_balance(struct lb_env *env)

kernel/sched/fair.c

11729

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

11731

if (asym_active_balance(env))

kernel/sched/fair.c

11734

if (imbalanced_active_balance(env))

kernel/sched/fair.c

11743

if (env->idle &&

kernel/sched/fair.c

11744

(env->src_rq->cfs.h_nr_runnable == 1)) {

kernel/sched/fair.c

11745

if ((check_cpu_capacity(env->src_rq, sd)) &&

kernel/sched/fair.c

11746

(capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))

kernel/sched/fair.c

11750

if (env->migration_type == migrate_misfit)

kernel/sched/fair.c

11758

static int should_we_balance(struct lb_env *env)

kernel/sched/fair.c

11761

struct sched_group *sg = env->sd->groups;

kernel/sched/fair.c

11768

if (!cpumask_test_cpu(env->dst_cpu, env->cpus))

kernel/sched/fair.c

11778

if (env->idle == CPU_NEWLY_IDLE) {

kernel/sched/fair.c

11779

if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)

kernel/sched/fair.c

11786

for_each_cpu_and(cpu, swb_cpus, env->cpus) {

kernel/sched/fair.c

11795

if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {

kernel/sched/fair.c

11813

return cpu == env->dst_cpu;

kernel/sched/fair.c

11818

return idle_smt == env->dst_cpu;

kernel/sched/fair.c

11821

return group_balance_cpu(sg) == env->dst_cpu;

kernel/sched/fair.c

11824

static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd,

kernel/sched/fair.c

11830

switch (env->migration_type) {

kernel/sched/fair.c

11832

__schedstat_add(sd->lb_imbalance_load[idle], env->imbalance);

kernel/sched/fair.c

11835

__schedstat_add(sd->lb_imbalance_util[idle], env->imbalance);

kernel/sched/fair.c

11838

__schedstat_add(sd->lb_imbalance_task[idle], env->imbalance);

kernel/sched/fair.c

11841

__schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);

kernel/sched/fair.c

11875

struct lb_env env = {

kernel/sched/fair.c

11884

.tasks = LIST_HEAD_INIT(env.tasks),

kernel/sched/fair.c

11893

if (!should_we_balance(&env)) {

kernel/sched/fair.c

11906

group = sched_balance_find_src_group(&env);

kernel/sched/fair.c

11912

busiest = sched_balance_find_src_rq(&env, group);

kernel/sched/fair.c

11918

WARN_ON_ONCE(busiest == env.dst_rq);

kernel/sched/fair.c

11920

update_lb_imbalance_stat(&env, sd, idle);

kernel/sched/fair.c

11922

env.src_cpu = busiest->cpu;

kernel/sched/fair.c

11923

env.src_rq = busiest;

kernel/sched/fair.c

11927

env.flags |= LBF_ALL_PINNED;

kernel/sched/fair.c

11935

env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);

kernel/sched/fair.c

11945

cur_ld_moved = detach_tasks(&env);

kernel/sched/fair.c

11958

attach_tasks(&env);

kernel/sched/fair.c

11964

if (env.flags & LBF_NEED_BREAK) {

kernel/sched/fair.c

11965

env.flags &= ~LBF_NEED_BREAK;

kernel/sched/fair.c

11988

if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {

kernel/sched/fair.c

11991

__cpumask_clear_cpu(env.dst_cpu, env.cpus);

kernel/sched/fair.c

11993

env.dst_rq = cpu_rq(env.new_dst_cpu);

kernel/sched/fair.c

11994

env.dst_cpu = env.new_dst_cpu;

kernel/sched/fair.c

11995

env.flags &= ~LBF_DST_PINNED;

kernel/sched/fair.c

11996

env.loop = 0;

kernel/sched/fair.c

11997

env.loop_break = SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

12012

if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)

kernel/sched/fair.c

12017

if (unlikely(env.flags & LBF_ALL_PINNED)) {

kernel/sched/fair.c

12027

if (!cpumask_subset(cpus, env.dst_grpmask)) {

kernel/sched/fair.c

12028

env.loop = 0;

kernel/sched/fair.c

12029

env.loop_break = SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

12048

env.migration_type != migrate_misfit)

kernel/sched/fair.c

12051

if (need_active_balance(&env)) {

kernel/sched/fair.c

12067

env.flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

12093

if (likely(!active_balance) || need_active_balance(&env)) {

kernel/sched/fair.c

12106

if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {

kernel/sched/fair.c

12136

if (env.idle == CPU_NEWLY_IDLE ||

kernel/sched/fair.c

12137

env.migration_type == migrate_misfit)

kernel/sched/fair.c

12141

if ((env.flags & LBF_ALL_PINNED &&

kernel/sched/fair.c

12238

struct lb_env env = {

kernel/sched/fair.c

12251

p = detach_one_task(&env);

kernel/sched/fair.c

2190

static void update_numa_stats(struct task_numa_env *env,

kernel/sched/fair.c

2211

!cpumask_test_cpu(cpu, env->p->cpus_ptr))

kernel/sched/fair.c

2224

ns->node_type = numa_classify(env->imbalance_pct, ns);

kernel/sched/fair.c

2230

static void task_numa_assign(struct task_numa_env *env,

kernel/sched/fair.c

2233

struct rq *rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2236

if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {

kernel/sched/fair.c

2238

int start = env->dst_cpu;

kernel/sched/fair.c

2241

for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {

kernel/sched/fair.c

2242

if (cpu == env->best_cpu || !idle_cpu(cpu) ||

kernel/sched/fair.c

2243

!cpumask_test_cpu(cpu, env->p->cpus_ptr)) {

kernel/sched/fair.c

2247

env->dst_cpu = cpu;

kernel/sched/fair.c

2248

rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2262

if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {

kernel/sched/fair.c

2263

rq = cpu_rq(env->best_cpu);

kernel/sched/fair.c

2267

if (env->best_task)

kernel/sched/fair.c

2268

put_task_struct(env->best_task);

kernel/sched/fair.c

2272

env->best_task = p;

kernel/sched/fair.c

2273

env->best_imp = imp;

kernel/sched/fair.c

2274

env->best_cpu = env->dst_cpu;

kernel/sched/fair.c

2278

struct task_numa_env *env)

kernel/sched/fair.c

2291

src_capacity = env->src_stats.compute_capacity;

kernel/sched/fair.c

2292

dst_capacity = env->dst_stats.compute_capacity;

kernel/sched/fair.c

2296

orig_src_load = env->src_stats.load;

kernel/sched/fair.c

2297

orig_dst_load = env->dst_stats.load;

kernel/sched/fair.c

2318

static bool task_numa_compare(struct task_numa_env *env,

kernel/sched/fair.c

2321

struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);

kernel/sched/fair.c

2322

struct rq *dst_rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2326

int dist = env->dist;

kernel/sched/fair.c

2344

if (cur == env->p) {

kernel/sched/fair.c

2350

if (maymove && moveimp >= env->best_imp)

kernel/sched/fair.c

2357

if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))

kernel/sched/fair.c

2364

if (env->best_task &&

kernel/sched/fair.c

2365

env->best_task->numa_preferred_nid == env->src_nid &&

kernel/sched/fair.c

2366

cur->numa_preferred_nid != env->src_nid) {

kernel/sched/fair.c

2388

if (env->dst_stats.node_type == node_has_spare)

kernel/sched/fair.c

2391

imp = taskimp + task_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

2392

task_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

2405

imp += group_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

2406

group_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

2408

imp += task_weight(cur, env->src_nid, dist) -

kernel/sched/fair.c

2409

task_weight(cur, env->dst_nid, dist);

kernel/sched/fair.c

2413

if (cur->numa_preferred_nid == env->dst_nid)

kernel/sched/fair.c

2422

if (cur->numa_preferred_nid == env->src_nid)

kernel/sched/fair.c

2425

if (maymove && moveimp > imp && moveimp > env->best_imp) {

kernel/sched/fair.c

2435

if (env->best_task && cur->numa_preferred_nid == env->src_nid &&

kernel/sched/fair.c

2436

env->best_task->numa_preferred_nid != env->src_nid) {

kernel/sched/fair.c

2446

if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)

kernel/sched/fair.c

2452

load = task_h_load(env->p) - task_h_load(cur);

kernel/sched/fair.c

2456

dst_load = env->dst_stats.load + load;

kernel/sched/fair.c

2457

src_load = env->src_stats.load - load;

kernel/sched/fair.c

2459

if (load_too_imbalanced(src_load, dst_load, env))

kernel/sched/fair.c

2465

int cpu = env->dst_stats.idle_cpu;

kernel/sched/fair.c

2469

cpu = env->dst_cpu;

kernel/sched/fair.c

2475

if (!idle_cpu(cpu) && env->best_cpu >= 0 &&

kernel/sched/fair.c

2476

idle_cpu(env->best_cpu)) {

kernel/sched/fair.c

2477

cpu = env->best_cpu;

kernel/sched/fair.c

2480

env->dst_cpu = cpu;

kernel/sched/fair.c

2483

task_numa_assign(env, cur, imp);

kernel/sched/fair.c

2490

if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))

kernel/sched/fair.c

2497

if (!maymove && env->best_task &&

kernel/sched/fair.c

2498

env->best_task->numa_preferred_nid == env->src_nid) {

kernel/sched/fair.c

2507

static void task_numa_find_cpu(struct task_numa_env *env,

kernel/sched/fair.c

2517

if (env->dst_stats.node_type == node_has_spare) {

kernel/sched/fair.c

2527

src_running = env->src_stats.nr_running - 1;

kernel/sched/fair.c

2528

dst_running = env->dst_stats.nr_running + 1;

kernel/sched/fair.c

2531

env->imb_numa_nr);

kernel/sched/fair.c

2536

if (env->dst_stats.idle_cpu >= 0) {

kernel/sched/fair.c

2537

env->dst_cpu = env->dst_stats.idle_cpu;

kernel/sched/fair.c

2538

task_numa_assign(env, NULL, 0);

kernel/sched/fair.c

2548

load = task_h_load(env->p);

kernel/sched/fair.c

2549

dst_load = env->dst_stats.load + load;

kernel/sched/fair.c

2550

src_load = env->src_stats.load - load;

kernel/sched/fair.c

2551

maymove = !load_too_imbalanced(src_load, dst_load, env);

kernel/sched/fair.c

2555

for_each_cpu_and(cpu, cpumask_of_node(env->dst_nid), env->p->cpus_ptr) {

kernel/sched/fair.c

2556

env->dst_cpu = cpu;

kernel/sched/fair.c

2557

if (task_numa_compare(env, taskimp, groupimp, maymove))

kernel/sched/fair.c

2564

struct task_numa_env env = {

kernel/sched/fair.c

2592

sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu));

kernel/sched/fair.c

2594

env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;

kernel/sched/fair.c

2595

env.imb_numa_nr = sd->imb_numa_nr;

kernel/sched/fair.c

2610

env.dst_nid = p->numa_preferred_nid;

kernel/sched/fair.c

2611

dist = env.dist = node_distance(env.src_nid, env.dst_nid);

kernel/sched/fair.c

2612

taskweight = task_weight(p, env.src_nid, dist);

kernel/sched/fair.c

2613

groupweight = group_weight(p, env.src_nid, dist);

kernel/sched/fair.c

2614

update_numa_stats(&env, &env.src_stats, env.src_nid, false);

kernel/sched/fair.c

2615

taskimp = task_weight(p, env.dst_nid, dist) - taskweight;

kernel/sched/fair.c

2616

groupimp = group_weight(p, env.dst_nid, dist) - groupweight;

kernel/sched/fair.c

2617

update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);

kernel/sched/fair.c

2620

task_numa_find_cpu(&env, taskimp, groupimp);

kernel/sched/fair.c

2630

if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {

kernel/sched/fair.c

2632

if (nid == env.src_nid || nid == p->numa_preferred_nid)

kernel/sched/fair.c

2635

dist = node_distance(env.src_nid, env.dst_nid);

kernel/sched/fair.c

2637

dist != env.dist) {

kernel/sched/fair.c

2638

taskweight = task_weight(p, env.src_nid, dist);

kernel/sched/fair.c

2639

groupweight = group_weight(p, env.src_nid, dist);

kernel/sched/fair.c

2648

env.dist = dist;

kernel/sched/fair.c

2649

env.dst_nid = nid;

kernel/sched/fair.c

2650

update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);

kernel/sched/fair.c

2651

task_numa_find_cpu(&env, taskimp, groupimp);

kernel/sched/fair.c

2664

if (env.best_cpu == -1)

kernel/sched/fair.c

2665

nid = env.src_nid;

kernel/sched/fair.c

2667

nid = cpu_to_node(env.best_cpu);

kernel/sched/fair.c

2674

if (env.best_cpu == -1) {

kernel/sched/fair.c

2675

trace_sched_stick_numa(p, env.src_cpu, NULL, -1);

kernel/sched/fair.c

2679

best_rq = cpu_rq(env.best_cpu);

kernel/sched/fair.c

2680

if (env.best_task == NULL) {

kernel/sched/fair.c

2681

ret = migrate_task_to(p, env.best_cpu);

kernel/sched/fair.c

2684

trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);

kernel/sched/fair.c

2688

ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);

kernel/sched/fair.c

2692

trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);

kernel/sched/fair.c

2693

put_task_struct(env.best_task);

kernel/sched/fair.c

9355

static int task_hot(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

9359

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

9368

if (env->sd->flags & SD_SHARE_CPUCAPACITY)

kernel/sched/fair.c

9374

if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&

kernel/sched/fair.c

9385

if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))

kernel/sched/fair.c

9391

delta = rq_clock_task(env->src_rq) - p->se.exec_start;

kernel/sched/fair.c

9402

static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

9411

if (!p->numa_faults || !(env->sd->flags & SD_NUMA))

kernel/sched/fair.c

9414

src_nid = cpu_to_node(env->src_cpu);

kernel/sched/fair.c

9415

dst_nid = cpu_to_node(env->dst_cpu);

kernel/sched/fair.c

9422

if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)

kernel/sched/fair.c

9433

if (env->idle == CPU_IDLE)

kernel/sched/fair.c

9450

struct lb_env *env)

kernel/sched/fair.c

9484

int can_migrate_task(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

9488

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

9501

if ((p->se.sched_delayed) && (env->migration_type != migrate_load))

kernel/sched/fair.c

9504

if (lb_throttled_hierarchy(p, env->dst_cpu))

kernel/sched/fair.c

9513

if (!env->sd->nr_balance_failed &&

kernel/sched/fair.c

9514

task_is_ineligible_on_dst_cpu(p, env->dst_cpu))

kernel/sched/fair.c

9524

if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {

kernel/sched/fair.c

9529

env->flags |= LBF_SOME_PINNED;

kernel/sched/fair.c

9541

if (env->idle == CPU_NEWLY_IDLE ||

kernel/sched/fair.c

9542

env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))

kernel/sched/fair.c

9546

cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr);

kernel/sched/fair.c

9549

env->flags |= LBF_DST_PINNED;

kernel/sched/fair.c

9550

env->new_dst_cpu = cpu;

kernel/sched/fair.c

9557

env->flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

9559

if (task_on_cpu(env->src_rq, p) ||

kernel/sched/fair.c

9560

task_current_donor(env->src_rq, p)) {

kernel/sched/fair.c

9572

if (env->flags & LBF_ACTIVE_LB)

kernel/sched/fair.c

9575

degrades = migrate_degrades_locality(p, env);

kernel/sched/fair.c

9577

hot = task_hot(p, env);

kernel/sched/fair.c

9581

if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) {

kernel/sched/fair.c

9594

static void detach_task(struct task_struct *p, struct lb_env *env)

kernel/sched/fair.c

9596

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

9600

schedstat_inc(env->sd->lb_hot_gained[env->idle]);

kernel/sched/fair.c

9604

WARN_ON(task_current(env->src_rq, p));

kernel/sched/fair.c

9605

WARN_ON(task_current_donor(env->src_rq, p));

kernel/sched/fair.c

9607

deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);

kernel/sched/fair.c

9608

set_task_cpu(p, env->dst_cpu);

kernel/sched/fair.c

9617

static struct task_struct *detach_one_task(struct lb_env *env)

kernel/sched/fair.c

9621

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

9624

&env->src_rq->cfs_tasks, se.group_node) {

kernel/sched/fair.c

9625

if (!can_migrate_task(p, env))

kernel/sched/fair.c

9628

detach_task(p, env);

kernel/sched/fair.c

9636

schedstat_inc(env->sd->lb_gained[env->idle]);

kernel/sched/fair.c

9648

static int detach_tasks(struct lb_env *env)

kernel/sched/fair.c

9650

struct list_head *tasks = &env->src_rq->cfs_tasks;

kernel/sched/fair.c

9655

lockdep_assert_rq_held(env->src_rq);

kernel/sched/fair.c

9661

if (env->src_rq->nr_running <= 1) {

kernel/sched/fair.c

9662

env->flags &= ~LBF_ALL_PINNED;

kernel/sched/fair.c

9666

if (env->imbalance <= 0)

kernel/sched/fair.c

9674

if (env->idle && env->src_rq->nr_running <= 1)

kernel/sched/fair.c

9677

env->loop++;

kernel/sched/fair.c

9679

if (env->loop > env->loop_max)

kernel/sched/fair.c

9683

if (env->loop > env->loop_break) {

kernel/sched/fair.c

9684

env->loop_break += SCHED_NR_MIGRATE_BREAK;

kernel/sched/fair.c

9685

env->flags |= LBF_NEED_BREAK;

kernel/sched/fair.c

9691

if (!can_migrate_task(p, env))

kernel/sched/fair.c

9694

switch (env->migration_type) {

kernel/sched/fair.c

9706

load < 16 && !env->sd->nr_balance_failed)

kernel/sched/fair.c

9715

if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)

kernel/sched/fair.c

9718

env->imbalance -= load;

kernel/sched/fair.c

9724

if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance)

kernel/sched/fair.c

9727

env->imbalance -= util;

kernel/sched/fair.c

9731

env->imbalance--;

kernel/sched/fair.c

9736

if (task_fits_cpu(p, env->src_cpu))

kernel/sched/fair.c

9739

env->imbalance = 0;

kernel/sched/fair.c

9743

detach_task(p, env);

kernel/sched/fair.c

9744

list_add(&p->se.group_node, &env->tasks);

kernel/sched/fair.c

9754

if (env->idle == CPU_NEWLY_IDLE)

kernel/sched/fair.c

9762

if (env->imbalance <= 0)

kernel/sched/fair.c

9778

schedstat_add(env->sd->lb_gained[env->idle], detached);

kernel/sched/fair.c

9813

static void attach_tasks(struct lb_env *env)

kernel/sched/fair.c

9815

struct list_head *tasks = &env->tasks;

kernel/sched/fair.c

9819

rq_lock(env->dst_rq, &rf);

kernel/sched/fair.c

9820

update_rq_clock(env->dst_rq);

kernel/sched/fair.c

9826

attach_task(env->dst_rq, p);

kernel/sched/fair.c

9829

rq_unlock(env->dst_rq, &rf);

lib/kobject_uevent.c

116

struct kobj_uevent_env *env = NULL;

lib/kobject_uevent.c

127

env = kzalloc_obj(*env);

lib/kobject_uevent.c

128

if (!env)

lib/kobject_uevent.c

133

add_uevent_var(env, "SYNTH_UUID=%.*s", UUID_STRING_LEN, buf))

lib/kobject_uevent.c

167

if (add_uevent_var(env, "SYNTH_ARG_%.*s=%.*s",

lib/kobject_uevent.c

175

kfree(env);

lib/kobject_uevent.c

177

*ret_env = env;

lib/kobject_uevent.c

196

struct kobj_uevent_env *env;

lib/kobject_uevent.c

212

count - (action_args - buf), &env);

lib/kobject_uevent.c

221

r = kobject_uevent_env(kobj, action, env->envp);

lib/kobject_uevent.c

222

kfree(env);

lib/kobject_uevent.c

251

static int init_uevent_argv(struct kobj_uevent_env *env, const char *subsystem)

lib/kobject_uevent.c

253

int buffer_size = sizeof(env->buf) - env->buflen;

lib/kobject_uevent.c

256

len = strscpy(&env->buf[env->buflen], subsystem, buffer_size);

lib/kobject_uevent.c

263

env->argv[0] = uevent_helper;

lib/kobject_uevent.c

264

env->argv[1] = &env->buf[env->buflen];

lib/kobject_uevent.c

265

env->argv[2] = NULL;

lib/kobject_uevent.c

267

env->buflen += len + 1;

lib/kobject_uevent.c

278

static struct sk_buff *alloc_uevent_skb(struct kobj_uevent_env *env,

lib/kobject_uevent.c

289

skb = alloc_skb(len + env->buflen, GFP_KERNEL);

lib/kobject_uevent.c

297

skb_put_data(skb, env->buf, env->buflen);

lib/kobject_uevent.c

308

static int uevent_net_broadcast_untagged(struct kobj_uevent_env *env,

lib/kobject_uevent.c

326

skb = alloc_uevent_skb(env, action_string, devpath);

lib/kobject_uevent.c

344

struct kobj_uevent_env *env,

lib/kobject_uevent.c

352

skb = alloc_uevent_skb(env, action_string, devpath);

lib/kobject_uevent.c

383

struct kobj_uevent_env *env,

lib/kobject_uevent.c

410

ret = uevent_net_broadcast_untagged(env, action_string,

lib/kobject_uevent.c

413

ret = uevent_net_broadcast_tagged(net->uevent_sock->sk, env,

lib/kobject_uevent.c

420

static void zap_modalias_env(struct kobj_uevent_env *env)

lib/kobject_uevent.c

426

for (i = 0; i < env->envp_idx;) {

lib/kobject_uevent.c

427

if (strncmp(env->envp[i], modalias_prefix,

lib/kobject_uevent.c

433

len = strlen(env->envp[i]) + 1;

lib/kobject_uevent.c

435

if (i != env->envp_idx - 1) {

lib/kobject_uevent.c

451

memmove(env->envp[i], env->envp[i + 1],

lib/kobject_uevent.c

452

env->buflen - (env->envp[i + 1] - env->envp[0]));

lib/kobject_uevent.c

454

for (j = i; j < env->envp_idx - 1; j++)

lib/kobject_uevent.c

455

env->envp[j] = env->envp[j + 1] - len;

lib/kobject_uevent.c

458

env->envp_idx--;

lib/kobject_uevent.c

459

env->buflen -= len;

lib/kobject_uevent.c

476

struct kobj_uevent_env *env;

lib/kobject_uevent.c

540

env = kzalloc_obj(struct kobj_uevent_env);

lib/kobject_uevent.c

541

if (!env)

lib/kobject_uevent.c

552

retval = add_uevent_var(env, "ACTION=%s", action_string);

lib/kobject_uevent.c

555

retval = add_uevent_var(env, "DEVPATH=%s", devpath);

lib/kobject_uevent.c

558

retval = add_uevent_var(env, "SUBSYSTEM=%s", subsystem);

lib/kobject_uevent.c

565

retval = add_uevent_var(env, "%s", envp_ext[i]);

lib/kobject_uevent.c

573

retval = uevent_ops->uevent(kobj, env);

lib/kobject_uevent.c

595

zap_modalias_env(env);

lib/kobject_uevent.c

603

retval = add_uevent_var(env, "SEQNUM=%llu",

lib/kobject_uevent.c

608

retval = kobject_uevent_net_broadcast(kobj, env, action_string,

lib/kobject_uevent.c

616

retval = add_uevent_var(env, "HOME=/");

lib/kobject_uevent.c

619

retval = add_uevent_var(env,

lib/kobject_uevent.c

623

retval = init_uevent_argv(env, subsystem);

lib/kobject_uevent.c

628

info = call_usermodehelper_setup(env->argv[0], env->argv,

lib/kobject_uevent.c

629

env->envp, GFP_KERNEL,

lib/kobject_uevent.c

630

NULL, cleanup_uevent_env, env);

lib/kobject_uevent.c

633

env = NULL; /* freed by cleanup_uevent_env */

lib/kobject_uevent.c

640

kfree(env);

lib/kobject_uevent.c

668

int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...)

lib/kobject_uevent.c

673

if (env->envp_idx >= ARRAY_SIZE(env->envp)) {

lib/kobject_uevent.c

679

len = vsnprintf(&env->buf[env->buflen],

lib/kobject_uevent.c

680

sizeof(env->buf) - env->buflen,

lib/kobject_uevent.c

684

if (len >= (sizeof(env->buf) - env->buflen)) {

lib/kobject_uevent.c

689

env->envp[env->envp_idx++] = &env->buf[env->buflen];

lib/kobject_uevent.c

690

env->buflen += len + 1;

net/atm/atm_sysfs.c

111

static int atm_uevent(const struct device *cdev, struct kobj_uevent_env *env)

net/atm/atm_sysfs.c

120

if (add_uevent_var(env, "NAME=%s%d", adev->type, adev->number))

net/core/net-sysfs.c

2218

static int netdev_uevent(const struct device *d, struct kobj_uevent_env *env)

net/core/net-sysfs.c

2224

retval = add_uevent_var(env, "INTERFACE=%s", dev->name);

net/core/net-sysfs.c

2232

retval = add_uevent_var(env, "IFINDEX=%d", dev->ifindex);

net/mac80211/parse.c

221

const struct ieee80211_tx_pwr_env *env = (const void *)data;

net/mac80211/parse.c

228

count = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_COUNT);

net/mac80211/parse.c

229

interpret = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_INTERPRET);

net/mac80211/parse.c

230

category = u8_get_bits(env->info, IEEE80211_TX_PWR_ENV_INFO_CATEGORY);

net/mac80211/parse.c

262

memcpy(out, env->variable, count + 1);

net/mac80211/parse.c

265

if (count == 3 && len > sizeof(*env) + count + 1) {

net/mac80211/parse.c

266

out[4] = env->variable[4];

net/mac80211/parse.c

273

memset(out, env->variable[0],

net/mac80211/parse.c

280

memcpy(out, env->variable, N);

net/mac80211/parse.c

284

if (len > sizeof(*env) + N) {

net/mac80211/parse.c

285

int K = u8_get_bits(env->variable[N],

net/mac80211/parse.c

289

memcpy(out + N, env->variable + N + 1, K);

net/rfkill/core.c

853

static int rfkill_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)

net/rfkill/core.c

861

error = add_uevent_var(env, "RFKILL_NAME=%s", rfkill->name);

net/rfkill/core.c

864

error = add_uevent_var(env, "RFKILL_TYPE=%s",

net/rfkill/core.c

872

error = add_uevent_var(env, "RFKILL_STATE=%d",

net/rfkill/core.c

876

return add_uevent_var(env, "RFKILL_HW_BLOCK_REASON=0x%lx", reasons);

net/wireless/core.h

56

enum environment_cap env;

net/wireless/reg.c

3344

enum environment_cap env = ENVIRON_ANY;

net/wireless/reg.c

3362

env = ENVIRON_INDOOR;

net/wireless/reg.c

3364

env = ENVIRON_OUTDOOR;

net/wireless/reg.c

3385

request->country_ie_env = env;

net/wireless/reg.c

568

char *env[] = { country, NULL };

net/wireless/reg.c

585

ret = kobject_uevent_env(&reg_fdev->dev.kobj, KOBJ_CHANGE, env);

scripts/kconfig/conf.c

189

char *env = getenv("KCONFIG_PROBABILITY");

scripts/kconfig/conf.c

192

while (env && *env) {

scripts/kconfig/conf.c

194

int tmp = strtol(env, &endp, 10);

scripts/kconfig/conf.c

203

env = (*endp == ':') ? endp + 1 : endp;

scripts/kconfig/conf.c

90

char *env;

scripts/kconfig/conf.c

93

env = getenv("KCONFIG_SEED");

scripts/kconfig/conf.c

94

if (env && *env) {

scripts/kconfig/conf.c

97

seed = strtol(env, &endp, 0);

scripts/kconfig/confdata.c

334

char *env;

scripts/kconfig/confdata.c

342

env = getenv("KCONFIG_DEFCONFIG_LIST");

scripts/kconfig/confdata.c

343

if (!env)

scripts/kconfig/confdata.c

349

while (isspace(*env))

scripts/kconfig/confdata.c

350

env++;

scripts/kconfig/confdata.c

352

if (!*env)

scripts/kconfig/confdata.c

355

p = env;

scripts/kconfig/confdata.c

363

name = env;

scripts/kconfig/confdata.c

375

env = p + 1;

scripts/kconfig/confdata.c

811

char *env;

scripts/kconfig/confdata.c

830

env = getenv("KCONFIG_OVERWRITECONFIG");

scripts/kconfig/confdata.c

831

if (env && *env) {

scripts/kconfig/gconf.c

1283

char *env = getenv(SRCTREE);

scripts/kconfig/gconf.c

1284

gchar *path = g_strconcat(env ? env : g_get_current_dir(), "/scripts/kconfig/icons/menu.xpm", NULL);

scripts/kconfig/gconf.c

1308

char *env;

scripts/kconfig/gconf.c

1315

env = getenv(SRCTREE);

scripts/kconfig/gconf.c

1316

if (env)

scripts/kconfig/gconf.c

1317

glade_file = g_strconcat(env, "/scripts/kconfig/gconf.ui", NULL);

scripts/kconfig/gconf.c

959

char *env = getenv(SRCTREE);

scripts/kconfig/gconf.c

960

gchar *path = g_strconcat(env ? env : g_get_current_dir(), "/scripts/kconfig/icons/", filename, NULL);

scripts/kconfig/preprocess.c

48

struct env *e;

scripts/kconfig/preprocess.c

57

static void env_del(struct env *e)

scripts/kconfig/preprocess.c

68

struct env *e;

scripts/kconfig/preprocess.c

94

struct env *e, *tmp;

security/tomoyo/common.c

1583

tomoyo_set_string(head, ptr->env->name);

security/tomoyo/common.h

1003

int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env) __must_hold_shared(&tomoyo_ss);

security/tomoyo/common.h

759

const struct tomoyo_path_info *env; /* environment variable */

security/tomoyo/environ.c

103

e.env = tomoyo_get_name(data);

security/tomoyo/environ.c

104

if (!e.env)

security/tomoyo/environ.c

108

tomoyo_put_name(e.env);

security/tomoyo/environ.c

24

return tomoyo_path_matches_pattern(r->param.environ.name, acl->env);

security/tomoyo/environ.c

51

int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env)

security/tomoyo/environ.c

56

if (!env || !*env)

security/tomoyo/environ.c

58

environ.name = env;

security/tomoyo/environ.c

83

return p1->env == p2->env;

security/tomoyo/gc.c

210

tomoyo_put_name(entry->env);

sound/aoa/soundbus/core.c

101

retval = add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);

sound/aoa/soundbus/core.c

104

retval = add_uevent_var(env, "MODALIAS=%s", soundbus_dev->modalias);

sound/aoa/soundbus/core.c

60

static int soundbus_uevent(const struct device *dev, struct kobj_uevent_env *env)

sound/aoa/soundbus/core.c

78

retval = add_uevent_var(env, "OF_NAME=%pOFn", of->dev.of_node);

sound/aoa/soundbus/core.c

82

retval = add_uevent_var(env, "OF_TYPE=%s", of_node_get_device_type(of->dev.of_node));

sound/aoa/soundbus/core.c

92

int tmp = env->buflen;

sound/aoa/soundbus/core.c

93

retval = add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);

sound/aoa/soundbus/core.c

96

compat += env->buflen - tmp;

sound/aoa/soundbus/core.c

97

cplen -= env->buflen - tmp;

sound/hda/core/hda_bus_type.c

68

static int hda_uevent(const struct device *dev, struct kobj_uevent_env *env)

sound/hda/core/hda_bus_type.c

74

if (add_uevent_var(env, "MODALIAS=%s", modalias))

tools/lib/subcmd/exec-cmd.c

121

char *env;

tools/lib/subcmd/exec-cmd.c

126

env = getenv(subcmd_config.exec_path_env);

tools/lib/subcmd/exec-cmd.c

127

if (env && *env)

tools/lib/subcmd/exec-cmd.c

128

return strdup(env);

tools/lib/subcmd/run-command.c

117

if (cmd->env) {

tools/lib/subcmd/run-command.c

118

for (; *cmd->env; cmd->env++) {

tools/lib/subcmd/run-command.c

119

if (strchr(*cmd->env, '='))

tools/lib/subcmd/run-command.c

120

putenv((char*)*cmd->env);

tools/lib/subcmd/run-command.c

122

unsetenv(*cmd->env);

tools/lib/subcmd/run-command.h

43

const char *const *env;

tools/objtool/builtin-check.c

118

char *env;

tools/objtool/builtin-check.c

121

env = getenv("OBJTOOL_ARGS");

tools/objtool/builtin-check.c

122

if (env) {

tools/objtool/builtin-check.c

125

envv[envc++] = env;

tools/objtool/builtin-check.c

126

env = strchr(env, ' ');

tools/objtool/builtin-check.c

127

if (!env)

tools/objtool/builtin-check.c

129

*env = '\0';

tools/objtool/builtin-check.c

130

env++;

tools/objtool/builtin-check.c

136

env = getenv("OBJTOOL_VERBOSE");

tools/objtool/builtin-check.c

137

if (env && !strcmp(env, "1"))

tools/perf/arch/common.c

109

char *env = getenv("PATH");

tools/perf/arch/common.c

111

if (!env)

tools/perf/arch/common.c

114

env = strdup(env);

tools/perf/arch/common.c

115

if (!env)

tools/perf/arch/common.c

118

path = strtok_r(env, ":", &tmp);

tools/perf/arch/common.c

127

free(env);

tools/perf/arch/common.c

144

static int perf_env__lookup_binutils_path(struct perf_env *env,

tools/perf/arch/common.c

148

const char *arch = perf_env__arch(env), *cross_env;

tools/perf/arch/common.c

221

int perf_env__lookup_objdump(struct perf_env *env, char **path)

tools/perf/arch/common.c

227

if (env->arch == NULL)

tools/perf/arch/common.c

230

return perf_env__lookup_binutils_path(env, "objdump", path);

tools/perf/arch/common.c

238

bool perf_env__single_address_space(struct perf_env *env)

tools/perf/arch/common.c

240

return strcmp(perf_env__arch(env), "sparc");

tools/perf/arch/common.h

10

bool perf_env__single_address_space(struct perf_env *env);

tools/perf/arch/common.h

9

int perf_env__lookup_objdump(struct perf_env *env, char **path);

tools/perf/builtin-c2c.c

2008

struct perf_env *env __maybe_unused)

tools/perf/builtin-c2c.c

202

struct perf_env *env);

tools/perf/builtin-c2c.c

2022

static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env)

tools/perf/builtin-c2c.c

2029

return sort_dimension__add(hpp_list, name, /*evlist=*/NULL, env, /*level=*/0);

tools/perf/builtin-c2c.c

2053

ret = _fn(hpp_list, tok, env); \

tools/perf/builtin-c2c.c

2067

struct perf_env *env)

tools/perf/builtin-c2c.c

208

struct perf_env *env)

tools/perf/builtin-c2c.c

2099

struct perf_env *env)

tools/perf/builtin-c2c.c

2113

return hpp_list__parse(&hists->list, /*output=*/NULL, sort, env);

tools/perf/builtin-c2c.c

2119

struct perf_env *env)

tools/perf/builtin-c2c.c

2122

return hpp_list__parse(&c2c_hists->list, output, sort, env);

tools/perf/builtin-c2c.c

222

ret = c2c_hists__init(hists, sort, nr_header_lines, env);

tools/perf/builtin-c2c.c

2257

struct perf_env *env = arg;

tools/perf/builtin-c2c.c

2271

c2c_hists__reinit(c2c_hists, c2c.cl_output, c2c.cl_resort, env);

tools/perf/builtin-c2c.c

2316

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-c2c.c

2321

c2c.nodes_cnt = env->nr_numa_nodes;

tools/perf/builtin-c2c.c

2322

c2c.cpus_cnt = env->nr_cpus_avail;

tools/perf/builtin-c2c.c

2324

n = env->numa_nodes;

tools/perf/builtin-c2c.c

2498

static void print_pareto(FILE *out, struct perf_env *env)

tools/perf/builtin-c2c.c

2523

ret = hpp_list__parse(&hpp_list, cl_output, /*evlist=*/NULL, env);

tools/perf/builtin-c2c.c

3127

struct perf_env *env;

tools/perf/builtin-c2c.c

3197

env = perf_session__env(session);

tools/perf/builtin-c2c.c

3204

if (!strcmp(perf_env__arch(env), "arm64"))

tools/perf/builtin-c2c.c

3234

err = mem2node__init(&c2c.mem2node, env);

tools/perf/builtin-c2c.c

3274

if (symbol__init(env) < 0)

tools/perf/builtin-c2c.c

391

c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2, machine->env);

tools/perf/builtin-record.c

2263

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-record.c

2270

env->clock.clockid_res_ns = rec->opts.clockid_res_ns;

tools/perf/builtin-record.c

2272

env->clock.clockid = rec->opts.clockid;

tools/perf/builtin-record.c

2287

env->clock.tod_ns = ref;

tools/perf/builtin-record.c

2292

env->clock.clockid_ns = ref;

tools/perf/builtin-record.c

2438

struct perf_env *env;

tools/perf/builtin-record.c

2480

env = perf_session__env(session);

tools/perf/builtin-record.c

2514

env->comp_type = PERF_COMP_ZSTD;

tools/perf/builtin-record.c

2515

env->comp_level = rec->opts.comp_level;

tools/perf/builtin-record.c

2568

env->comp_mmap_len = session->evlist->core.mmap_len;

tools/perf/builtin-record.c

2898

env->comp_ratio = ratio + 0.5;

tools/perf/builtin-sched.c

3293

struct perf_env *env;

tools/perf/builtin-sched.c

3318

env = perf_session__env(session);

tools/perf/builtin-sched.c

3327

symbol__init(env);

tools/perf/builtin-sched.c

3366

sched->max_cpu.cpu = env->nr_cpus_online;

tools/perf/builtin-sched.c

4525

cd_map = session->header.env.cpu_domain;

tools/perf/builtin-sched.c

4579

cd_map0 = session[0]->header.env.cpu_domain;

tools/perf/builtin-sched.c

4596

cd_map1 = session[1]->header.env.cpu_domain;

tools/perf/builtin-script.c

2488

struct cgroup *cgrp = cgroup__find(machine->env,

tools/perf/builtin-script.c

4196

struct perf_env *env;

tools/perf/builtin-script.c

4445

env = perf_session__env(session);

tools/perf/builtin-script.c

4455

if (symbol__init(env) < 0)

tools/perf/builtin-script.c

747

struct perf_env *env;

tools/perf/builtin-script.c

757

env = perf_session__env(script->session);

tools/perf/builtin-script.c

758

if (!env->clock.enabled) {

tools/perf/builtin-script.c

763

clockid_ns = env->clock.clockid_ns;

tools/perf/builtin-script.c

764

tod_ns = env->clock.tod_ns;

tools/perf/builtin-stat.c

1623

struct perf_env *env = data;

tools/perf/builtin-stat.c

1627

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1634

struct perf_env *env = data;

tools/perf/builtin-stat.c

1643

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1644

id.die = env->cpu[cpu.cpu].die_id;

tools/perf/builtin-stat.c

1650

static void perf_env__get_cache_id_for_cpu(struct perf_cpu cpu, struct perf_env *env,

tools/perf/builtin-stat.c

1654

int caches_cnt = env->caches_cnt;

tools/perf/builtin-stat.c

1655

struct cpu_cache_level *caches = env->caches;

tools/perf/builtin-stat.c

1690

struct perf_env *env = data;

tools/perf/builtin-stat.c

1696

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1697

id.die = env->cpu[cpu.cpu].die_id;

tools/perf/builtin-stat.c

1698

perf_env__get_cache_id_for_cpu(cpu, env, cache_level, &id);

tools/perf/builtin-stat.c

1707

struct perf_env *env = data;

tools/perf/builtin-stat.c

1711

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1712

id.die = env->cpu[cpu.cpu].die_id;

tools/perf/builtin-stat.c

1713

id.cluster = env->cpu[cpu.cpu].cluster_id;

tools/perf/builtin-stat.c

1721

struct perf_env *env = data;

tools/perf/builtin-stat.c

1729

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1730

id.die = env->cpu[cpu.cpu].die_id;

tools/perf/builtin-stat.c

1731

id.cluster = env->cpu[cpu.cpu].cluster_id;

tools/perf/builtin-stat.c

1732

id.core = env->cpu[cpu.cpu].core_id;

tools/perf/builtin-stat.c

1740

struct perf_env *env = data;

tools/perf/builtin-stat.c

1749

id.socket = env->cpu[cpu.cpu].socket_id;

tools/perf/builtin-stat.c

1750

id.die = env->cpu[cpu.cpu].die_id;

tools/perf/builtin-stat.c

1751

id.core = env->cpu[cpu.cpu].core_id;

tools/perf/builtin-stat.c

1879

struct perf_env *env = perf_session__env(st->session);

tools/perf/builtin-stat.c

1903

get_id, env, needs_sort);

tools/perf/builtin-stat.c

2187

struct perf_env *env = perf_session__env(session);

tools/perf/builtin-stat.c

2188

const char **argv = env->cmdline_argv;

tools/perf/builtin-stat.c

2189

int argc = env->nr_cmdline;

tools/perf/builtin-timechart.c

1522

tchart->numcpus = ph->env.nr_cpus_avail;

tools/perf/builtin-timechart.c

1529

if (svg_build_topology_map(&ph->env))

tools/perf/builtin-trace.c

3008

struct perf_env *env = evsel__env(evsel) ?: &trace->host_env;

tools/perf/builtin-trace.c

3010

const char *e = perf_env__arch_strerrno(env, err);

tools/perf/builtin-trace.c

4890

fprintf(fp, "\t\t\t\t%s: %d\n", perf_env__arch_strerrno(trace->host->env, e + 1), stats->errnos[e]);

tools/perf/tests/hists_cumulate.c

298

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

445

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

503

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_cumulate.c

687

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_filter.c

145

if (setup_sorting(evlist, machine->env) < 0)

tools/perf/tests/hists_link.c

317

if (setup_sorting(evlist, machine->env) < 0)

tools/perf/tests/hists_output.c

149

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

251

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

307

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

381

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/hists_output.c

483

setup_sorting(/*evlist=*/NULL, machine->env);

tools/perf/tests/mem2node.c

51

struct perf_env env = {

tools/perf/tests/mem2node.c

66

T("failed: mem2node__init", !mem2node__init(&map, &env));

tools/perf/tests/topology.c

101

if (!env->cpu && strncmp(env->arch, "s390", 4) && strncmp(env->arch, "aarch64", 7))

tools/perf/tests/topology.c

110

if (!strncmp(env->arch, "ppc64le", 7)) {

tools/perf/tests/topology.c

115

TEST_ASSERT_VAL("Session header CPU map not set", env->cpu);

tools/perf/tests/topology.c

117

for (i = 0; i < env->nr_cpus_avail; i++) {

tools/perf/tests/topology.c

122

env->cpu[i].core_id,

tools/perf/tests/topology.c

123

env->cpu[i].socket_id);

tools/perf/tests/topology.c

133

env->cpu[cpu.cpu].core_id == id.core);

tools/perf/tests/topology.c

135

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

138

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

147

env->cpu[cpu.cpu].core_id == id.core);

tools/perf/tests/topology.c

150

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

153

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

162

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

165

env->cpu[cpu.cpu].die_id == id.die);

tools/perf/tests/topology.c

177

env->cpu[cpu.cpu].socket_id == id.socket);

tools/perf/tests/topology.c

73

struct perf_env *env;

tools/perf/tests/topology.c

77

env = perf_session__env(session);

tools/perf/ui/browsers/hists.c

2295

struct perf_env *env)

tools/perf/ui/browsers/hists.c

2301

browser->env = env;

tools/perf/ui/browsers/hists.c

2475

perf_env__lookup_objdump(browser->env, &annotate_opts.objdump_path))

tools/perf/ui/browsers/hists.c

3001

struct perf_env *env, bool warn_lost_event)

tools/perf/ui/browsers/hists.c

3004

struct hist_browser *browser = perf_evsel_browser__new(evsel, hbt, env);

tools/perf/ui/browsers/hists.c

3236

if (env->arch)

tools/perf/ui/browsers/hists.c

3460

struct perf_env *env;

tools/perf/ui/browsers/hists.c

3557

menu->min_pcnt, menu->env,

tools/perf/ui/browsers/hists.c

3615

struct hist_browser_timer *hbt, float min_pcnt, struct perf_env *env,

tools/perf/ui/browsers/hists.c

3630

.env = env,

tools/perf/ui/browsers/hists.c

3665

float min_pcnt, struct perf_env *env, bool warn_lost_event)

tools/perf/ui/browsers/hists.c

3674

env, warn_lost_event);

tools/perf/ui/browsers/hists.c

3691

return __evlist__tui_browse_hists(evlist, nr_entries, help, hbt, min_pcnt, env,

tools/perf/ui/browsers/hists.c

3711

float min_percent, struct perf_env *env)

tools/perf/ui/browsers/hists.c

3729

browser->env = env;

tools/perf/ui/browsers/hists.h

16

struct perf_env *env;

tools/perf/util/amd-sample-raw.c

336

static void parse_cpuid(struct perf_env *env)

tools/perf/util/amd-sample-raw.c

341

cpuid = perf_env__cpuid(env);

tools/perf/util/amd-sample-raw.c

357

struct perf_env *env = perf_session__env(evlist->session);

tools/perf/util/amd-sample-raw.c

358

int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env);

tools/perf/util/amd-sample-raw.c

359

const char *pmu_mapping = perf_env__pmu_mappings(env);

tools/perf/util/amd-sample-raw.c

374

if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions"))

tools/perf/util/amd-sample-raw.c

377

if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat"))

tools/perf/util/amd-sample-raw.c

380

if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize"))

tools/perf/util/amd-sample-raw.c

385

parse_cpuid(env);

tools/perf/util/annotate.c

1000

arch = arch__find(e_machine, e_flags, machine->env ? machine->env->cpuid : NULL);

tools/perf/util/auxtrace.c

1913

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/auxtrace.c

1923

int kernel_is_64_bit = perf_env__kernel_is_64_bit(env);

tools/perf/util/auxtrace.c

2025

struct perf_env *env, const struct perf_tool *tool,

tools/perf/util/auxtrace.c

2028

return __auxtrace_mmap__read(map, itr, env, tool, fn, false, 0);

tools/perf/util/auxtrace.c

2032

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/auxtrace.c

2036

return __auxtrace_mmap__read(map, itr, env, tool, fn, true, snapshot_size);

tools/perf/util/auxtrace.h

510

struct perf_env *env, const struct perf_tool *tool,

tools/perf/util/auxtrace.h

514

struct auxtrace_record *itr, struct perf_env *env,

tools/perf/util/block-info.c

503

struct evsel *evsel, struct perf_env *env)

tools/perf/util/block-info.c

515

ret = block_hists_tui_browse(bh, evsel, min_percent, env);

tools/perf/util/block-info.h

72

struct evsel *evsel, struct perf_env *env);

tools/perf/util/bpf-event.c

104

static int perf_env__fetch_btf(struct perf_env *env,

tools/perf/util/bpf-event.c

122

if (!perf_env__insert_btf(env, node)) {

tools/perf/util/bpf-event.c

48

struct perf_env *env = machine->env;

tools/perf/util/bpf-event.c

507

struct perf_env *env = &session->header.env;

tools/perf/util/bpf-event.c

514

perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb,

tools/perf/util/bpf-event.c

54

if (env == NULL)

tools/perf/util/bpf-event.c

543

struct perf_env *env;

tools/perf/util/bpf-event.c

552

env = perf_session__env(session);

tools/perf/util/bpf-event.c

57

info_node = perf_env__find_bpf_prog_info(env, id);

tools/perf/util/bpf-event.c

604

perf_env__fetch_btf(env, info->btf_id, btf);

tools/perf/util/bpf-event.c

659

if (!perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/bpf-event.c

73

dso__bpf_prog(dso)->env = env;

tools/perf/util/bpf-event.c

834

static int perf_env__add_bpf_info(struct perf_env *env, u32 id)

tools/perf/util/bpf-event.c

869

if (!perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/bpf-event.c

890

perf_env__fetch_btf(env, btf_id, btf);

tools/perf/util/bpf-event.c

901

struct perf_env *env = data;

tools/perf/util/bpf-event.c

909

ret = perf_env__add_bpf_info(env, event->bpf.id);

tools/perf/util/bpf-event.c

926

int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)

tools/perf/util/bpf-event.c

943

return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);

tools/perf/util/bpf-event.c

947

struct perf_env *env,

tools/perf/util/bpf-event.c

964

node = __perf_env__find_btf(env, info->btf_id);

tools/perf/util/bpf-event.h

42

int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env);

tools/perf/util/bpf-event.h

44

struct perf_env *env,

tools/perf/util/bpf-event.h

56

struct perf_env *env __maybe_unused)

tools/perf/util/bpf-event.h

62

struct perf_env *env __maybe_unused,

tools/perf/util/cgroup.c

558

struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,

tools/perf/util/cgroup.c

563

down_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

564

cgrp = __cgroup__findnew(&env->cgroups.tree, id, true, path);

tools/perf/util/cgroup.c

565

up_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

574

struct cgroup *cgroup__find(struct perf_env *env, uint64_t id)

tools/perf/util/cgroup.c

578

down_read(&env->cgroups.lock);

tools/perf/util/cgroup.c

579

cgrp = __cgroup__findnew(&env->cgroups.tree, id, false, NULL);

tools/perf/util/cgroup.c

580

up_read(&env->cgroups.lock);

tools/perf/util/cgroup.c

584

void perf_env__purge_cgroups(struct perf_env *env)

tools/perf/util/cgroup.c

589

down_write(&env->cgroups.lock);

tools/perf/util/cgroup.c

590

while (!RB_EMPTY_ROOT(&env->cgroups.tree)) {

tools/perf/util/cgroup.c

591

node = rb_first(&env->cgroups.tree);

tools/perf/util/cgroup.c

594

rb_erase(node, &env->cgroups.tree);

tools/perf/util/cgroup.c

597

up_write(&env->cgroups.lock);

tools/perf/util/cgroup.h

37

struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id,

tools/perf/util/cgroup.h

39

struct cgroup *cgroup__find(struct perf_env *env, uint64_t id);

tools/perf/util/cgroup.h

42

void perf_env__purge_cgroups(struct perf_env *env);

tools/perf/util/data-convert-bt.c

1354

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1361

ncpus = env->nr_cpus_avail ?: MAX_CPUS;

tools/perf/util/data-convert-bt.c

1387

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1396

ADD("host", env->hostname);

tools/perf/util/data-convert-bt.c

1398

ADD("release", env->os_release);

tools/perf/util/data-convert-bt.c

1399

ADD("version", env->version);

tools/perf/util/data-convert-bt.c

1400

ADD("machine", env->arch);

tools/perf/util/data-convert-bt.c

1425

if (session->header.env.hostname) {

tools/perf/util/data-convert-bt.c

1427

session->header.env.hostname);

tools/perf/util/data-convert-bt.c

1431

if (session->header.env.os_release) {

tools/perf/util/data-convert-bt.c

1433

session->header.env.os_release);

tools/perf/util/data-convert-bt.c

1437

if (session->header.env.version) {

tools/perf/util/data-convert-bt.c

1439

session->header.env.version);

tools/perf/util/data-convert-bt.c

1443

if (session->header.env.arch) {

tools/perf/util/data-convert-bt.c

1445

session->header.env.arch);

tools/perf/util/data-convert-bt.c

1463

struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-bt.c

1465

if (!env->clock.enabled) {

tools/perf/util/data-convert-bt.c

1471

desc = clockid_name(env->clock.clockid);

tools/perf/util/data-convert-bt.c

1472

offset = env->clock.tod_ns - env->clock.clockid_ns;

tools/perf/util/data-convert-json.c

276

const struct perf_env *env = perf_session__env(session);

tools/perf/util/data-convert-json.c

299

output_json_key_string(out, true, 2, "hostname", env->hostname);

tools/perf/util/data-convert-json.c

300

output_json_key_string(out, true, 2, "os-release", env->os_release);

tools/perf/util/data-convert-json.c

301

output_json_key_string(out, true, 2, "arch", env->arch);

tools/perf/util/data-convert-json.c

303

if (env->cpu_desc)

tools/perf/util/data-convert-json.c

304

output_json_key_string(out, true, 2, "cpu-desc", env->cpu_desc);

tools/perf/util/data-convert-json.c

306

output_json_key_string(out, true, 2, "cpuid", env->cpuid);

tools/perf/util/data-convert-json.c

307

output_json_key_format(out, true, 2, "nrcpus-online", "%u", env->nr_cpus_online);

tools/perf/util/data-convert-json.c

308

output_json_key_format(out, true, 2, "nrcpus-avail", "%u", env->nr_cpus_avail);

tools/perf/util/data-convert-json.c

310

if (env->clock.enabled) {

tools/perf/util/data-convert-json.c

312

"%u", env->clock.clockid);

tools/perf/util/data-convert-json.c

314

"%" PRIu64, env->clock.clockid_ns);

tools/perf/util/data-convert-json.c

316

"%" PRIu64, env->clock.tod_ns);

tools/perf/util/data-convert-json.c

319

output_json_key_string(out, true, 2, "perf-version", env->version);

tools/perf/util/data-convert-json.c

322

for (i = 0; i < env->nr_cmdline; i++) {

tools/perf/util/data-convert-json.c

324

output_json_string(c->out, env->cmdline_argv[i]);

tools/perf/util/dso.c

1968

info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env,

tools/perf/util/dso.c

861

node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);

tools/perf/util/dso.c

883

node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id);

tools/perf/util/dso.h

247

struct perf_env *env;

tools/perf/util/env.c

100

first = rb_first(&env->bpf_progs.infos);

tools/perf/util/env.c

103

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

106

bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)

tools/perf/util/env.c

110

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

111

ret = __perf_env__insert_btf(env, btf_node);

tools/perf/util/env.c

112

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

116

bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node)

tools/perf/util/env.c

123

p = &env->bpf_progs.btfs.rb_node;

tools/perf/util/env.c

139

rb_insert_color(&btf_node->rb_node, &env->bpf_progs.btfs);

tools/perf/util/env.c

140

env->bpf_progs.btfs_cnt++;

tools/perf/util/env.c

144

struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id)

tools/perf/util/env.c

148

down_read(&env->bpf_progs.lock);

tools/perf/util/env.c

149

res = __perf_env__find_btf(env, btf_id);

tools/perf/util/env.c

150

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

154

struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id)

tools/perf/util/env.c

159

n = env->bpf_progs.btfs.rb_node;

tools/perf/util/env.c

174

static void perf_env__purge_bpf(struct perf_env *env)

tools/perf/util/env.c

179

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

181

root = &env->bpf_progs.infos;

tools/perf/util/env.c

195

env->bpf_progs.infos_cnt = 0;

tools/perf/util/env.c

197

root = &env->bpf_progs.btfs;

tools/perf/util/env.c

209

env->bpf_progs.btfs_cnt = 0;

tools/perf/util/env.c

211

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

214

static void perf_env__purge_bpf(struct perf_env *env __maybe_unused)

tools/perf/util/env.c

247

void perf_env__exit(struct perf_env *env)

tools/perf/util/env.c

251

perf_env__purge_bpf(env);

tools/perf/util/env.c

252

perf_env__purge_cgroups(env);

tools/perf/util/env.c

253

zfree(&env->hostname);

tools/perf/util/env.c

254

zfree(&env->os_release);

tools/perf/util/env.c

255

zfree(&env->version);

tools/perf/util/env.c

256

zfree(&env->arch);

tools/perf/util/env.c

257

zfree(&env->cpu_desc);

tools/perf/util/env.c

258

zfree(&env->cpuid);

tools/perf/util/env.c

259

zfree(&env->cmdline);

tools/perf/util/env.c

260

zfree(&env->cmdline_argv);

tools/perf/util/env.c

261

zfree(&env->sibling_dies);

tools/perf/util/env.c

262

zfree(&env->sibling_cores);

tools/perf/util/env.c

263

zfree(&env->sibling_threads);

tools/perf/util/env.c

264

zfree(&env->pmu_mappings);

tools/perf/util/env.c

265

zfree(&env->cpu);

tools/perf/util/env.c

266

for (i = 0; i < env->nr_cpu_pmu_caps; i++)

tools/perf/util/env.c

267

zfree(&env->cpu_pmu_caps[i]);

tools/perf/util/env.c

268

zfree(&env->cpu_pmu_caps);

tools/perf/util/env.c

269

zfree(&env->numa_map);

tools/perf/util/env.c

27

bool perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

271

for (i = 0; i < env->nr_numa_nodes; i++)

tools/perf/util/env.c

272

perf_cpu_map__put(env->numa_nodes[i].map);

tools/perf/util/env.c

273

zfree(&env->numa_nodes);

tools/perf/util/env.c

275

for (i = 0; i < env->caches_cnt; i++)

tools/perf/util/env.c

276

cpu_cache_level__free(&env->caches[i]);

tools/perf/util/env.c

277

zfree(&env->caches);

tools/perf/util/env.c

279

for (i = 0; i < env->nr_memory_nodes; i++)

tools/perf/util/env.c

280

zfree(&env->memory_nodes[i].set);

tools/perf/util/env.c

281

zfree(&env->memory_nodes);

tools/perf/util/env.c

283

for (i = 0; i < env->nr_hybrid_nodes; i++) {

tools/perf/util/env.c

284

zfree(&env->hybrid_nodes[i].pmu_name);

tools/perf/util/env.c

285

zfree(&env->hybrid_nodes[i].cpus);

tools/perf/util/env.c

287

zfree(&env->hybrid_nodes);

tools/perf/util/env.c

289

for (i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/env.c

290

for (j = 0; j < env->pmu_caps[i].nr_caps; j++)

tools/perf/util/env.c

291

zfree(&env->pmu_caps[i].caps[j]);

tools/perf/util/env.c

292

zfree(&env->pmu_caps[i].caps);

tools/perf/util/env.c

293

zfree(&env->pmu_caps[i].pmu_name);

tools/perf/util/env.c

295

zfree(&env->pmu_caps);

tools/perf/util/env.c

296

free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail);

tools/perf/util/env.c

299

void perf_env__init(struct perf_env *env)

tools/perf/util/env.c

301

memset(env, 0, sizeof(*env));

tools/perf/util/env.c

303

env->bpf_progs.infos = RB_ROOT;

tools/perf/util/env.c

304

env->bpf_progs.btfs = RB_ROOT;

tools/perf/util/env.c

305

init_rwsem(&env->bpf_progs.lock);

tools/perf/util/env.c

307

env->kernel_is_64_bit = -1;

tools/perf/util/env.c

310

static void perf_env__init_kernel_mode(struct perf_env *env)

tools/perf/util/env.c

312

const char *arch = perf_env__raw_arch(env);

tools/perf/util/env.c

318

env->kernel_is_64_bit = 1;

tools/perf/util/env.c

32

down_write(&env->bpf_progs.lock);

tools/perf/util/env.c

320

env->kernel_is_64_bit = 0;

tools/perf/util/env.c

323

int perf_env__kernel_is_64_bit(struct perf_env *env)

tools/perf/util/env.c

325

if (env->kernel_is_64_bit == -1)

tools/perf/util/env.c

326

perf_env__init_kernel_mode(env);

tools/perf/util/env.c

328

return env->kernel_is_64_bit;

tools/perf/util/env.c

33

ret = __perf_env__insert_bpf_prog_info(env, info_node);

tools/perf/util/env.c

331

int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[])

tools/perf/util/env.c

336

env->cmdline_argv = calloc(argc, sizeof(char *));

tools/perf/util/env.c

337

if (env->cmdline_argv == NULL)

tools/perf/util/env.c

34

up_write(&env->bpf_progs.lock);

tools/perf/util/env.c

345

env->cmdline_argv[i] = argv[i];

tools/perf/util/env.c

346

if (env->cmdline_argv[i] == NULL)

tools/perf/util/env.c

350

env->nr_cmdline = argc;

tools/perf/util/env.c

354

zfree(&env->cmdline_argv);

tools/perf/util/env.c

359

int perf_env__read_cpu_topology_map(struct perf_env *env)

tools/perf/util/env.c

363

if (env->cpu != NULL)

tools/perf/util/env.c

366

if (env->nr_cpus_avail == 0)

tools/perf/util/env.c

367

env->nr_cpus_avail = cpu__max_present_cpu().cpu;

tools/perf/util/env.c

369

nr_cpus = env->nr_cpus_avail;

tools/perf/util/env.c

373

env->cpu = calloc(nr_cpus, sizeof(env->cpu[0]));

tools/perf/util/env.c

374

if (env->cpu == NULL)

tools/perf/util/env.c

383

env->cpu[idx].core_id = core_id >= 0 ? core_id : -1;

tools/perf/util/env.c

384

env->cpu[idx].socket_id = socket_id >= 0 ? socket_id : -1;

tools/perf/util/env.c

385

env->cpu[idx].die_id = die_id >= 0 ? die_id : -1;

tools/perf/util/env.c

388

env->nr_cpus_avail = nr_cpus;

tools/perf/util/env.c

39

bool __perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node)

tools/perf/util/env.c

392

int perf_env__read_pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

405

env->nr_pmu_mappings = pmu_num;

tools/perf/util/env.c

418

env->pmu_mappings = strbuf_detach(&sb, NULL);

tools/perf/util/env.c

427

int perf_env__read_cpuid(struct perf_env *env)

tools/perf/util/env.c

436

free(env->cpuid);

tools/perf/util/env.c

437

env->cpuid = strdup(cpuid);

tools/perf/util/env.c

438

if (env->cpuid == NULL)

tools/perf/util/env.c

443

static int perf_env__read_arch(struct perf_env *env)

tools/perf/util/env.c

447

if (env->arch)

tools/perf/util/env.c

451

env->arch = strdup(uts.machine);

tools/perf/util/env.c

453

return env->arch ? 0 : -ENOMEM;

tools/perf/util/env.c

456

static int perf_env__read_nr_cpus_avail(struct perf_env *env)

tools/perf/util/env.c

458

if (env->nr_cpus_avail == 0)

tools/perf/util/env.c

459

env->nr_cpus_avail = cpu__max_present_cpu().cpu;

tools/perf/util/env.c

46

p = &env->bpf_progs.infos.rb_node;

tools/perf/util/env.c

461

return env->nr_cpus_avail ? 0 : -ENOENT;

tools/perf/util/env.c

510

int perf_env__read_core_pmu_caps(struct perf_env *env)

tools/perf/util/env.c

529

return __perf_env__read_core_pmu_caps(pmu, &env->nr_cpu_pmu_caps,

tools/perf/util/env.c

530

&env->cpu_pmu_caps,

tools/perf/util/env.c

531

&env->max_branches,

tools/perf/util/env.c

532

&env->br_cntr_nr,

tools/perf/util/env.c

533

&env->br_cntr_width);

tools/perf/util/env.c

559

env->nr_pmus_with_caps = nr_pmu;

tools/perf/util/env.c

560

env->pmu_caps = pmu_caps;

tools/perf/util/env.c

574

const char *perf_env__raw_arch(struct perf_env *env)

tools/perf/util/env.c

576

return env && !perf_env__read_arch(env) ? env->arch : "unknown";

tools/perf/util/env.c

579

int perf_env__nr_cpus_avail(struct perf_env *env)

tools/perf/util/env.c

581

return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0;

tools/perf/util/env.c

62

rb_insert_color(&info_node->rb_node, &env->bpf_progs.infos);

tools/perf/util/env.c

623

const char *perf_env__arch(struct perf_env *env)

tools/perf/util/env.c

627

if (!env || !env->arch) { /* Assume local operation */

tools/perf/util/env.c

63

env->bpf_progs.infos_cnt++;

tools/perf/util/env.c

633

arch_name = env->arch;

tools/perf/util/env.c

642

const char *perf_env__arch_strerrno(struct perf_env *env __maybe_unused, int err __maybe_unused)

tools/perf/util/env.c

645

if (env->arch_strerrno == NULL)

tools/perf/util/env.c

646

env->arch_strerrno = arch_syscalls__strerrno_function(perf_env__arch(env));

tools/perf/util/env.c

648

return env->arch_strerrno ? env->arch_strerrno(err) : "no arch specific strerrno function";

tools/perf/util/env.c

654

const char *perf_env__cpuid(struct perf_env *env)

tools/perf/util/env.c

658

if (!env->cpuid) { /* Assume local operation */

tools/perf/util/env.c

659

status = perf_env__read_cpuid(env);

tools/perf/util/env.c

664

return env->cpuid;

tools/perf/util/env.c

667

int perf_env__nr_pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

67

struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

671

if (!env->nr_pmu_mappings) { /* Assume local operation */

tools/perf/util/env.c

672

status = perf_env__read_pmu_mappings(env);

tools/perf/util/env.c

677

return env->nr_pmu_mappings;

tools/perf/util/env.c

680

const char *perf_env__pmu_mappings(struct perf_env *env)

tools/perf/util/env.c

684

if (!env->pmu_mappings) { /* Assume local operation */

tools/perf/util/env.c

685

status = perf_env__read_pmu_mappings(env);

tools/perf/util/env.c

690

return env->pmu_mappings;

tools/perf/util/env.c

693

int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu)

tools/perf/util/env.c

695

if (!env->nr_numa_map) {

tools/perf/util/env.c

699

for (i = 0; i < env->nr_numa_nodes; i++) {

tools/perf/util/env.c

700

nn = &env->numa_nodes[i];

tools/perf/util/env.c

710

env->numa_map = malloc(nr * sizeof(int));

tools/perf/util/env.c

711

if (!env->numa_map)

tools/perf/util/env.c

715

env->numa_map[i] = -1;

tools/perf/util/env.c

717

env->nr_numa_map = nr;

tools/perf/util/env.c

719

for (i = 0; i < env->nr_numa_nodes; i++) {

tools/perf/util/env.c

723

nn = &env->numa_nodes[i];

tools/perf/util/env.c

725

env->numa_map[tmp.cpu] = i;

tools/perf/util/env.c

729

return cpu.cpu >= 0 && cpu.cpu < env->nr_numa_map ? env->numa_map[cpu.cpu] : -1;

tools/perf/util/env.c

73

down_read(&env->bpf_progs.lock);

tools/perf/util/env.c

732

bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name)

tools/perf/util/env.c

734

char *pmu_mapping = env->pmu_mappings, *colon;

tools/perf/util/env.c

736

for (int i = 0; i < env->nr_pmu_mappings; ++i) {

tools/perf/util/env.c

74

n = env->bpf_progs.infos.rb_node;

tools/perf/util/env.c

750

char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,

tools/perf/util/env.c

770

for (i = 0; i < env->nr_cpu_pmu_caps; i++) {

tools/perf/util/env.c

771

if (!strncmp(env->cpu_pmu_caps[i], cap_eq, cap_size + 1)) {

tools/perf/util/env.c

773

return &env->cpu_pmu_caps[i][cap_size + 1];

tools/perf/util/env.c

779

for (i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/env.c

780

if (strcmp(env->pmu_caps[i].pmu_name, pmu_name))

tools/perf/util/env.c

783

ptr = env->pmu_caps[i].caps;

tools/perf/util/env.c

785

for (j = 0; j < env->pmu_caps[i].nr_caps; j++) {

tools/perf/util/env.c

798

void perf_env__find_br_cntr_info(struct perf_env *env,

tools/perf/util/env.c

803

*nr = env->cpu_pmu_caps ? env->br_cntr_nr :

tools/perf/util/env.c

804

env->pmu_caps->br_cntr_nr;

tools/perf/util/env.c

808

*width = env->cpu_pmu_caps ? env->br_cntr_width :

tools/perf/util/env.c

809

env->pmu_caps->br_cntr_width;

tools/perf/util/env.c

813

bool perf_env__is_x86_amd_cpu(struct perf_env *env)

tools/perf/util/env.c

818

is_amd = env->cpuid && strstarts(env->cpuid, "AuthenticAMD") ? 1 : -1;

tools/perf/util/env.c

825

struct perf_env env = { .total_mem = 0, };

tools/perf/util/env.c

828

perf_env__cpuid(&env);

tools/perf/util/env.c

829

is_amd = perf_env__is_x86_amd_cpu(&env);

tools/perf/util/env.c

830

perf_env__exit(&env);

tools/perf/util/env.c

835

bool perf_env__is_x86_intel_cpu(struct perf_env *env)

tools/perf/util/env.c

840

is_intel = env->cpuid && strstarts(env->cpuid, "GenuineIntel") ? 1 : -1;

tools/perf/util/env.c

847

struct perf_env env = { .total_mem = 0, };

tools/perf/util/env.c

850

perf_env__cpuid(&env);

tools/perf/util/env.c

851

is_intel = perf_env__is_x86_intel_cpu(&env);

tools/perf/util/env.c

852

perf_env__exit(&env);

tools/perf/util/env.c

88

up_read(&env->bpf_progs.lock);

tools/perf/util/env.c

92

void perf_env__iterate_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.c

99

down_read(&env->bpf_progs.lock);

tools/perf/util/env.h

172

int perf_env__read_core_pmu_caps(struct perf_env *env);

tools/perf/util/env.h

174

void perf_env__exit(struct perf_env *env);

tools/perf/util/env.h

176

int perf_env__kernel_is_64_bit(struct perf_env *env);

tools/perf/util/env.h

178

int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]);

tools/perf/util/env.h

180

int perf_env__read_cpuid(struct perf_env *env);

tools/perf/util/env.h

181

int perf_env__read_pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

182

int perf_env__nr_pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

183

const char *perf_env__pmu_mappings(struct perf_env *env);

tools/perf/util/env.h

185

int perf_env__read_cpu_topology_map(struct perf_env *env);

tools/perf/util/env.h

189

const char *perf_env__arch(struct perf_env *env);

tools/perf/util/env.h

190

const char *perf_env__arch_strerrno(struct perf_env *env, int err);

tools/perf/util/env.h

191

const char *perf_env__cpuid(struct perf_env *env);

tools/perf/util/env.h

192

const char *perf_env__raw_arch(struct perf_env *env);

tools/perf/util/env.h

193

int perf_env__nr_cpus_avail(struct perf_env *env);

tools/perf/util/env.h

195

void perf_env__init(struct perf_env *env);

tools/perf/util/env.h

197

bool __perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

199

bool perf_env__insert_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

201

struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

203

void perf_env__iterate_bpf_prog_info(struct perf_env *env,

tools/perf/util/env.h

207

bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);

tools/perf/util/env.h

208

bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node);

tools/perf/util/env.h

209

struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id);

tools/perf/util/env.h

210

struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id);

tools/perf/util/env.h

213

int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu);

tools/perf/util/env.h

214

char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,

tools/perf/util/env.h

217

bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name);

tools/perf/util/env.h

218

void perf_env__find_br_cntr_info(struct perf_env *env,

tools/perf/util/env.h

223

bool perf_env__is_x86_amd_cpu(struct perf_env *env);

tools/perf/util/env.h

225

bool perf_env__is_x86_intel_cpu(struct perf_env *env);

tools/perf/util/event.c

838

struct perf_env *env = machine->env;

tools/perf/util/event.c

840

if (env && env->cpu)

tools/perf/util/event.c

841

al->socket = env->cpu[al->cpu].socket_id;

tools/perf/util/header.c

1033

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

1038

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1040

ret = do_write(ff, &env->bpf_progs.infos_cnt,

tools/perf/util/header.c

1041

sizeof(env->bpf_progs.infos_cnt));

tools/perf/util/header.c

1042

if (ret < 0 || env->bpf_progs.infos_cnt == 0)

tools/perf/util/header.c

1045

root = &env->bpf_progs.infos;

tools/perf/util/header.c

1068

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1075

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

1080

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1082

ret = do_write(ff, &env->bpf_progs.btfs_cnt,

tools/perf/util/header.c

1083

sizeof(env->bpf_progs.btfs_cnt));

tools/perf/util/header.c

1085

if (ret < 0 || env->bpf_progs.btfs_cnt == 0)

tools/perf/util/header.c

1088

root = &env->bpf_progs.btfs;

tools/perf/util/header.c

1101

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1522

ret = do_write(ff, &(ff->ph->env.comp_ver), sizeof(ff->ph->env.comp_ver));

tools/perf/util/header.c

1526

ret = do_write(ff, &(ff->ph->env.comp_type), sizeof(ff->ph->env.comp_type));

tools/perf/util/header.c

1530

ret = do_write(ff, &(ff->ph->env.comp_level), sizeof(ff->ph->env.comp_level));

tools/perf/util/header.c

1534

ret = do_write(ff, &(ff->ph->env.comp_ratio), sizeof(ff->ph->env.comp_ratio));

tools/perf/util/header.c

1538

return do_write(ff, &(ff->ph->env.comp_mmap_len), sizeof(ff->ph->env.comp_mmap_len));

tools/perf/util/header.c

1790

fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname);

tools/perf/util/header.c

1795

fprintf(fp, "# os release : %s\n", ff->ph->env.os_release);

tools/perf/util/header.c

1800

fprintf(fp, "# arch : %s\n", ff->ph->env.arch);

tools/perf/util/header.c

1805

fprintf(fp, "# e_machine : %u\n", ff->ph->env.e_machine);

tools/perf/util/header.c

1806

fprintf(fp, "# e_flags : %u\n", ff->ph->env.e_flags);

tools/perf/util/header.c

1811

fprintf(fp, "# cpudesc : %s\n", ff->ph->env.cpu_desc);

tools/perf/util/header.c

1816

fprintf(fp, "# nrcpus online : %u\n", ff->ph->env.nr_cpus_online);

tools/perf/util/header.c

1817

fprintf(fp, "# nrcpus avail : %u\n", ff->ph->env.nr_cpus_avail);

tools/perf/util/header.c

1822

fprintf(fp, "# perf version : %s\n", ff->ph->env.version);

tools/perf/util/header.c

1829

nr = ff->ph->env.nr_cmdline;

tools/perf/util/header.c

1834

char *argv_i = strdup(ff->ph->env.cmdline_argv[i]);

tools/perf/util/header.c

1836

fprintf(fp, "%s ", ff->ph->env.cmdline_argv[i]);

tools/perf/util/header.c

1857

int cpu_nr = ph->env.nr_cpus_avail;

tools/perf/util/header.c

1861

nr = ph->env.nr_sibling_cores;

tools/perf/util/header.c

1862

str = ph->env.sibling_cores;

tools/perf/util/header.c

1869

if (ph->env.nr_sibling_dies) {

tools/perf/util/header.c

1870

nr = ph->env.nr_sibling_dies;

tools/perf/util/header.c

1871

str = ph->env.sibling_dies;

tools/perf/util/header.c

1879

nr = ph->env.nr_sibling_threads;

tools/perf/util/header.c

1880

str = ph->env.sibling_threads;

tools/perf/util/header.c

1887

if (ph->env.nr_sibling_dies) {

tools/perf/util/header.c

1888

if (ph->env.cpu != NULL) {

tools/perf/util/header.c

1892

i, ph->env.cpu[i].core_id,

tools/perf/util/header.c

1893

ph->env.cpu[i].die_id,

tools/perf/util/header.c

1894

ph->env.cpu[i].socket_id);

tools/perf/util/header.c

1899

if (ph->env.cpu != NULL) {

tools/perf/util/header.c

1903

i, ph->env.cpu[i].core_id,

tools/perf/util/header.c

1904

ph->env.cpu[i].socket_id);

tools/perf/util/header.c

1914

ff->ph->env.clock.clockid_res_ns * 1000);

tools/perf/util/header.c

1926

if (!ff->ph->env.clock.enabled) {

tools/perf/util/header.c

1932

ref = ff->ph->env.clock.tod_ns;

tools/perf/util/header.c

1938

ref = ff->ph->env.clock.clockid_ns;

tools/perf/util/header.c

1943

clockid = ff->ph->env.clock.clockid;

tools/perf/util/header.c

1966

for (i = 0; i < ff->ph->env.nr_hybrid_nodes; i++) {

tools/perf/util/header.c

1967

n = &ff->ph->env.hybrid_nodes[i];

tools/perf/util/header.c

1986

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

1990

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

1992

root = &env->bpf_progs.infos;

tools/perf/util/header.c

2005

env, fp);

tools/perf/util/header.c

2008

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2013

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2017

down_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2019

root = &env->bpf_progs.btfs;

tools/perf/util/header.c

2033

up_read(&env->bpf_progs.lock);

tools/perf/util/header.c

2213

fprintf(fp, "# total memory : %llu kB\n", ff->ph->env.total_mem);

tools/perf/util/header.c

2221

for (i = 0; i < ff->ph->env.nr_numa_nodes; i++) {

tools/perf/util/header.c

2222

n = &ff->ph->env.numa_nodes[i];

tools/perf/util/header.c

2235

fprintf(fp, "# cpuid : %s\n", ff->ph->env.cpuid);

tools/perf/util/header.c

2258

for (i = 0; i < ff->ph->env.caches_cnt; i++) {

tools/perf/util/header.c

2260

cpu_cache_level__fprintf(fp, &ff->ph->env.caches[i]);

tools/perf/util/header.c

2267

ff->ph->env.comp_type == PERF_COMP_ZSTD ? "Zstd" : "Unknown",

tools/perf/util/header.c

2268

ff->ph->env.comp_level, ff->ph->env.comp_ratio);

tools/perf/util/header.c

2292

__print_pmu_caps(fp, ff->ph->env.nr_cpu_pmu_caps,

tools/perf/util/header.c

2293

ff->ph->env.cpu_pmu_caps, (char *)"cpu");

tools/perf/util/header.c

2298

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2301

for (int i = 0; i < env->nr_pmus_with_caps; i++) {

tools/perf/util/header.c

2302

pmu_caps = &env->pmu_caps[i];

tools/perf/util/header.c

2307

if (strcmp(perf_env__arch(env), "x86") == 0 &&

tools/perf/util/header.c

2308

perf_env__has_pmu_mapping(env, "ibs_op")) {

tools/perf/util/header.c

2309

char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise");

tools/perf/util/header.c

2318

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2324

pmu_num = env->nr_pmu_mappings;

tools/perf/util/header.c

2330

str = env->pmu_mappings;

tools/perf/util/header.c

2412

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2416

nodes = env->memory_nodes;

tools/perf/util/header.c

2417

nr = env->nr_memory_nodes;

tools/perf/util/header.c

2420

nr, env->memory_bsize);

tools/perf/util/header.c

2423

memory_node__fprintf(&nodes[i], env->memory_bsize, fp);

tools/perf/util/header.c

2429

struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain;

tools/perf/util/header.c

2430

u32 nr = ff->ph->env.nr_cpus_avail;

tools/perf/util/header.c

2434

fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version);

tools/perf/util/header.c

2435

fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains);

tools/perf/util/header.c

2451

if (ff->ph->env.schedstat_version >= 17)

tools/perf/util/header.c

2624

free(ff->ph->env.__feat_env); \

tools/perf/util/header.c

2625

ff->ph->env.__feat_env = do_read_string(ff); \

tools/perf/util/header.c

2626

return ff->ph->env.__feat_env ? 0 : -ENOMEM; \

tools/perf/util/header.c

2640

ret = do_read_u32(ff, &ff->ph->env.e_machine);

tools/perf/util/header.c

2644

return do_read_u32(ff, &ff->ph->env.e_flags);

tools/perf/util/header.c

2665

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2676

env->nr_cpus_avail = (int)nr_cpus_avail;

tools/perf/util/header.c

2677

env->nr_cpus_online = (int)nr_cpus_online;

tools/perf/util/header.c

2683

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2690

env->total_mem = (unsigned long long)total_mem;

tools/perf/util/header.c

2751

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2758

env->nr_cmdline = nr;

tools/perf/util/header.c

2778

env->cmdline = cmdline;

tools/perf/util/header.c

2779

env->cmdline_argv = (const char **) argv;

tools/perf/util/header.c

2793

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2794

int cpu_nr = env->nr_cpus_avail;

tools/perf/util/header.c

2797

env->cpu = calloc(cpu_nr, sizeof(*env->cpu));

tools/perf/util/header.c

2798

if (!env->cpu)

tools/perf/util/header.c

2804

env->nr_sibling_cores = nr;

tools/perf/util/header.c

2820

env->sibling_cores = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

2825

env->nr_sibling_threads = nr;

tools/perf/util/header.c

2839

env->sibling_threads = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

2846

zfree(&env->cpu);

tools/perf/util/header.c

2854

env->cpu[i].core_id = nr;

tools/perf/util/header.c

2860

env->cpu[i].socket_id = nr;

tools/perf/util/header.c

2874

env->nr_sibling_dies = nr;

tools/perf/util/header.c

2888

env->sibling_dies = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

2894

env->cpu[i].die_id = nr;

tools/perf/util/header.c

2903

zfree(&env->cpu);

tools/perf/util/header.c

2909

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2944

env->nr_numa_nodes = nr;

tools/perf/util/header.c

2945

env->numa_nodes = nodes;

tools/perf/util/header.c

2955

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

2969

env->nr_pmu_mappings = pmu_num;

tools/perf/util/header.c

2988

env->msr_pmu_type = type;

tools/perf/util/header.c

2994

free(env->pmu_mappings);

tools/perf/util/header.c

2995

env->pmu_mappings = strbuf_detach(&sb, NULL);

tools/perf/util/header.c

3005

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3019

env->nr_groups = nr_groups;

tools/perf/util/header.c

3103

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3144

env->caches = caches;

tools/perf/util/header.c

3145

env->caches_cnt = cnt;

tools/perf/util/header.c

3181

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3220

env->memory_bsize = bsize;

tools/perf/util/header.c

3221

env->memory_nodes = nodes;

tools/perf/util/header.c

3222

env->nr_memory_nodes = nr;

tools/perf/util/header.c

3234

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3236

if (do_read_u64(ff, &env->clock.clockid_res_ns))

tools/perf/util/header.c

3245

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3260

env->clock.clockid = data32;

tools/perf/util/header.c

3266

env->clock.tod_ns = data64;

tools/perf/util/header.c

3272

env->clock.clockid_ns = data64;

tools/perf/util/header.c

3273

env->clock.enabled = true;

tools/perf/util/header.c

3280

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3304

env->nr_hybrid_nodes = nr;

tools/perf/util/header.c

3305

env->hybrid_nodes = nodes;

tools/perf/util/header.c

3337

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3350

down_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3392

if (!__perf_env__insert_bpf_prog_info(env, info_node)) {

tools/perf/util/header.c

3398

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3403

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3409

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3422

down_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3442

if (!__perf_env__insert_btf(env, node))

tools/perf/util/header.c

3449

up_write(&env->bpf_progs.lock);

tools/perf/util/header.c

3458

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3460

if (do_read_u32(ff, &(env->comp_ver)))

tools/perf/util/header.c

3463

if (do_read_u32(ff, &(env->comp_type)))

tools/perf/util/header.c

3466

if (do_read_u32(ff, &(env->comp_level)))

tools/perf/util/header.c

3469

if (do_read_u32(ff, &(env->comp_ratio)))

tools/perf/util/header.c

3472

if (do_read_u32(ff, &(env->comp_mmap_len)))

tools/perf/util/header.c

3544

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3545

int ret = __process_pmu_caps(ff, &env->nr_cpu_pmu_caps,

tools/perf/util/header.c

3546

&env->cpu_pmu_caps,

tools/perf/util/header.c

3547

&env->max_branches,

tools/perf/util/header.c

3548

&env->br_cntr_nr,

tools/perf/util/header.c

3549

&env->br_cntr_width);

tools/perf/util/header.c

3551

if (!ret && !env->cpu_pmu_caps)

tools/perf/util/header.c

3558

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3596

env->nr_pmus_with_caps = nr_pmu;

tools/perf/util/header.c

3597

env->pmu_caps = pmu_caps;

tools/perf/util/header.c

3615

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

3622

nra = env->nr_cpus_avail;

tools/perf/util/header.c

3623

nr = env->nr_cpus_online;

tools/perf/util/header.c

3629

env->cpu_domain = cd_map;

tools/perf/util/header.c

3635

env->schedstat_version = schedstat_version;

tools/perf/util/header.c

3641

env->max_sched_domains = max_sched_domains;

tools/perf/util/header.c

4554

session->machines.host.env = &header->env;

tools/perf/util/header.c

575

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

583

n = env->nr_cmdline + 1;

tools/perf/util/header.c

593

for (i = 0 ; i < env->nr_cmdline; i++) {

tools/perf/util/header.c

594

ret = do_write_string(ff, env->cmdline_argv[i]);

tools/perf/util/header.c

605

struct perf_env *env = &ff->ph->env;

tools/perf/util/header.c

633

ret = perf_env__read_cpu_topology_map(env);

tools/perf/util/header.c

637

for (j = 0; j < env->nr_cpus_avail; j++) {

tools/perf/util/header.c

638

ret = do_write(ff, &env->cpu[j].core_id,

tools/perf/util/header.c

639

sizeof(env->cpu[j].core_id));

tools/perf/util/header.c

642

ret = do_write(ff, &env->cpu[j].socket_id,

tools/perf/util/header.c

643

sizeof(env->cpu[j].socket_id));

tools/perf/util/header.c

661

for (j = 0; j < env->nr_cpus_avail; j++) {

tools/perf/util/header.c

662

ret = do_write(ff, &env->cpu[j].die_id,

tools/perf/util/header.c

663

sizeof(env->cpu[j].die_id));

tools/perf/util/header.c

942

return do_write(ff, &ff->ph->env.clock.clockid_res_ns,

tools/perf/util/header.c

943

sizeof(ff->ph->env.clock.clockid_res_ns));

tools/perf/util/header.c

961

data32 = ff->ph->env.clock.clockid;

tools/perf/util/header.c

968

data64 = &ff->ph->env.clock.tod_ns;

tools/perf/util/header.c

975

data64 = &ff->ph->env.clock.clockid_ns;

tools/perf/util/header.h

112

struct perf_env env;

tools/perf/util/hist.c

254

struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(h->ms.thread))->env,

tools/perf/util/hist.h

726

float min_pcnt, struct perf_env *env, bool warn_lost_event);

tools/perf/util/hist.h

736

float min_percent, struct perf_env *env);

tools/perf/util/hist.h

743

struct perf_env *env __maybe_unused,

tools/perf/util/hist.h

784

struct perf_env *env __maybe_unused)

tools/perf/util/intel-pt.c

1297

struct perf_env *env = pt->machine->env;

tools/perf/util/intel-pt.c

1383

if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))

tools/perf/util/intel-pt.c

3570

struct perf_env *env = pt->machine->env;

tools/perf/util/intel-pt.c

3573

if (!env->cpuid)

tools/perf/util/intel-pt.c

3584

if (!strncmp(env->cpuid, "GenuineIntel,6,190,", 19))

tools/perf/util/intel-pt.c

3587

if (!strncmp(env->cpuid, "GenuineIntel,6,175,", 19) ||

tools/perf/util/intel-pt.c

3588

!strncmp(env->cpuid, "GenuineIntel,6,182,", 19))

tools/perf/util/libbfd.c

546

info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env,

tools/perf/util/libbfd.c

564

node = perf_env__find_btf(dso__bpf_prog(dso)->env,

tools/perf/util/lock-contention.c

107

const char *arch = perf_env__arch(machine->env);

tools/perf/util/machine.c

145

machine->env = host_env;

tools/perf/util/machine.c

2802

struct perf_env *env = evsel__env(evsel);

tools/perf/util/machine.c

2806

!env ? 0 : env->max_branches,

tools/perf/util/machine.c

3150

return machine && !strcmp(perf_env__raw_arch(machine->env), arch);

tools/perf/util/machine.c

3155

return machine && !strcmp(perf_env__arch(machine->env), arch);

tools/perf/util/machine.c

3160

return machine ? perf_env__nr_cpus_avail(machine->env) : 0;

tools/perf/util/machine.c

667

cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);

tools/perf/util/machine.c

90

machine->env = NULL;

tools/perf/util/machine.h

44

struct perf_env *env;

tools/perf/util/mem2node.c

47

int mem2node__init(struct mem2node *map, struct perf_env *env)

tools/perf/util/mem2node.c

49

struct memory_node *n, *nodes = &env->memory_nodes[0];

tools/perf/util/mem2node.c

51

u64 bsize = env->memory_bsize;

tools/perf/util/mem2node.c

57

for (i = 0; i < env->nr_memory_nodes; i++) {

tools/perf/util/mem2node.c

66

for (i = 0; i < env->nr_memory_nodes; i++) {

tools/perf/util/mem2node.h

16

int mem2node__init(struct mem2node *map, struct perf_env *env);

tools/perf/util/parse-events.c

583

struct perf_env env = {};

tools/perf/util/parse-events.c

585

perf_env__init(&env);

tools/perf/util/parse-events.c

586

len = perf_env__kernel_is_64_bit(&env) ? sizeof(u64) : sizeof(long);

tools/perf/util/parse-events.c

587

perf_env__exit(&env);

tools/perf/util/sample-raw.c

15

void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sample-raw.c

17

const char *arch_pf = perf_env__arch(env);

tools/perf/util/sample-raw.c

18

const char *cpuid = perf_env__cpuid(env);

tools/perf/util/sample-raw.h

14

void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env);

tools/perf/util/session.c

160

perf_env__init(&session->header.env);

tools/perf/util/session.c

182

evlist__init_trace_event_sample_raw(session->evlist, &session->header.env);

tools/perf/util/session.c

197

session->machines.host.env = host_env;

tools/perf/util/session.c

203

perf_env__single_address_space(session->machines.host.env);

tools/perf/util/session.c

256

perf_env__exit(&session->header.env);

tools/perf/util/session.c

2964

return &session->header.env;

tools/perf/util/session.c

2989

struct perf_env *env;

tools/perf/util/session.c

2999

env = perf_session__env(session);

tools/perf/util/session.c

3000

if (env && env->e_machine != EM_NONE) {

tools/perf/util/session.c

3002

*e_flags = env->e_flags;

tools/perf/util/session.c

3004

return env->e_machine;

tools/perf/util/session.c

3012

if (env) {

tools/perf/util/session.c

3013

env->e_machine = args.e_machine;

tools/perf/util/session.c

3014

env->e_flags = args.e_flags;

tools/perf/util/sort.c

1019

struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(he->ms.thread))->env,

tools/perf/util/sort.c

2540

static int arch_support_sort_key(const char *sort_key, struct perf_env *env)

tools/perf/util/sort.c

2542

const char *arch = perf_env__arch(env);

tools/perf/util/sort.c

2553

static const char *arch_perf_header_entry(const char *se_header, struct perf_env *env)

tools/perf/util/sort.c

2555

const char *arch = perf_env__arch(env);

tools/perf/util/sort.c

2575

static void sort_dimension_add_dynamic_header(struct sort_dimension *sd, struct perf_env *env)

tools/perf/util/sort.c

2577

sd->entry->se_header = arch_perf_header_entry(sd->entry->se_header, env);

tools/perf/util/sort.c

3682

struct evlist *evlist, struct perf_env *env,

tools/perf/util/sort.c

3695

!arch_support_sort_key(tok, env)) {

tools/perf/util/sort.c

3708

sort_dimension_add_dynamic_header(sd, env);

tools/perf/util/sort.c

3762

static bool is_hpp_sort_key(const char *key, struct perf_env *env)

tools/perf/util/sort.c

3768

!arch_support_sort_key(key, env)) {

tools/perf/util/sort.c

3790

struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

3819

if (is_hpp_sort_key(tok, env)) {

tools/perf/util/sort.c

3832

ret = sort_dimension__add(list, tok, evlist, env, level);

tools/perf/util/sort.c

3961

static int __setup_sorting(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

4001

ret = setup_sort_list(&perf_hpp_list, str, evlist, env);

tools/perf/util/sort.c

4237

int setup_sorting(struct evlist *evlist, struct perf_env *env)

tools/perf/util/sort.c

4241

err = __setup_sorting(evlist, env);

tools/perf/util/sort.c

4246

err = sort_dimension__add(&perf_hpp_list, "parent", evlist, env, -1);

tools/perf/util/sort.h

134

int setup_sorting(struct evlist *evlist, struct perf_env *env);

tools/perf/util/sort.h

149

struct evlist *evlist, struct perf_env *env,

tools/perf/util/svghelper.c

751

int svg_build_topology_map(struct perf_env *env)

tools/perf/util/svghelper.c

758

nr_cpus = min(env->nr_cpus_online, MAX_NR_CPUS);

tools/perf/util/svghelper.c

760

t.sib_core_nr = env->nr_sibling_cores;

tools/perf/util/svghelper.c

761

t.sib_thr_nr = env->nr_sibling_threads;

tools/perf/util/svghelper.c

762

t.sib_core = calloc(env->nr_sibling_cores, sizeof(cpumask_t));

tools/perf/util/svghelper.c

763

t.sib_thr = calloc(env->nr_sibling_threads, sizeof(cpumask_t));

tools/perf/util/svghelper.c

765

sib_core = env->sibling_cores;

tools/perf/util/svghelper.c

766

sib_thr = env->sibling_threads;

tools/perf/util/svghelper.c

773

for (i = 0; i < env->nr_sibling_cores; i++) {

tools/perf/util/svghelper.c

782

for (i = 0; i < env->nr_sibling_threads; i++) {

tools/perf/util/svghelper.h

33

int svg_build_topology_map(struct perf_env *env);

tools/perf/util/symbol.c

2207

static int vmlinux_path__init(struct perf_env *env)

tools/perf/util/symbol.c

2227

if (env) {

tools/perf/util/symbol.c

2228

kernel_version = env->os_release;

tools/perf/util/symbol.c

2392

int symbol__init(struct perf_env *env)

tools/perf/util/symbol.c

2403

if (symbol_conf.try_vmlinux_path && vmlinux_path__init(env) < 0)

tools/perf/util/symbol.h

153

int symbol__init(struct perf_env *env);

tools/perf/util/unwind-libunwind.c

36

if (!machine->env || !machine->env->arch)

tools/perf/util/unwind-libunwind.c

43

arch = perf_env__arch(machine->env);

tools/testing/selftests/bpf/bench.c

152

hits_mean, hits_stddev, hits_mean / env.producer_cnt);

tools/testing/selftests/bpf/bench.c

16

struct env env = {

tools/testing/selftests/bpf/bench.c

164

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/bench.c

188

hits_mean, hits_stddev, hits_mean / env.producer_cnt);

tools/testing/selftests/bpf/bench.c

189

printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);

tools/testing/selftests/bpf/bench.c

28

if (level == LIBBPF_DEBUG && !env.verbose)

tools/testing/selftests/bpf/bench.c

315

env.verbose = true;

tools/testing/selftests/bpf/bench.c

318

env.list = true;

tools/testing/selftests/bpf/bench.c

321

env.duration_sec = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

322

if (env.duration_sec <= 0) {

tools/testing/selftests/bpf/bench.c

328

env.warmup_sec = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

329

if (env.warmup_sec <= 0) {

tools/testing/selftests/bpf/bench.c

335

env.producer_cnt = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

336

if (env.producer_cnt < 0) {

tools/testing/selftests/bpf/bench.c

342

env.consumer_cnt = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/bench.c

343

if (env.consumer_cnt < 0) {

tools/testing/selftests/bpf/bench.c

349

env.affinity = true;

tools/testing/selftests/bpf/bench.c

352

env.quiet = true;

tools/testing/selftests/bpf/bench.c

355

env.stacktrace = true;

tools/testing/selftests/bpf/bench.c

358

env.affinity = true;

tools/testing/selftests/bpf/bench.c

359

if (parse_num_list(arg, &env.prod_cpus.cpus,

tools/testing/selftests/bpf/bench.c

360

&env.prod_cpus.cpus_len)) {

tools/testing/selftests/bpf/bench.c

366

env.affinity = true;

tools/testing/selftests/bpf/bench.c

367

if (parse_num_list(arg, &env.cons_cpus.cpus,

tools/testing/selftests/bpf/bench.c

368

&env.cons_cpus.cpus_len)) {

tools/testing/selftests/bpf/bench.c

379

env.bench_name = strdup(arg);

tools/testing/selftests/bpf/bench.c

487

return cpu_set->next_cpu++ % env.nr_cpus;

tools/testing/selftests/bpf/bench.c

658

if (!env.bench_name) {

tools/testing/selftests/bpf/bench.c

663

if (strcmp(benchs[i]->name, env.bench_name) == 0) {

tools/testing/selftests/bpf/bench.c

669

fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);

tools/testing/selftests/bpf/bench.c

678

if (!env.quiet)

tools/testing/selftests/bpf/bench.c

681

state.producers = calloc(env.producer_cnt, sizeof(*state.producers));

tools/testing/selftests/bpf/bench.c

682

state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));

tools/testing/selftests/bpf/bench.c

683

state.results = calloc(env.duration_sec + env.warmup_sec + 2,

tools/testing/selftests/bpf/bench.c

693

for (i = 0; i < env.consumer_cnt; i++) {

tools/testing/selftests/bpf/bench.c

705

if (env.affinity)

tools/testing/selftests/bpf/bench.c

707

next_cpu(&env.cons_cpus));

tools/testing/selftests/bpf/bench.c

713

if (!env.prod_cpus.cpus)

tools/testing/selftests/bpf/bench.c

714

env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;

tools/testing/selftests/bpf/bench.c

716

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/bench.c

728

if (env.affinity)

tools/testing/selftests/bpf/bench.c

730

next_cpu(&env.prod_cpus));

tools/testing/selftests/bpf/bench.c

733

if (!env.quiet)

tools/testing/selftests/bpf/bench.c

74

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/bench.c

749

if (iter == env.duration_sec + env.warmup_sec) {

tools/testing/selftests/bpf/bench.c

758

env.nr_cpus = get_nprocs();

tools/testing/selftests/bpf/bench.c

761

if (env.list) {

tools/testing/selftests/bpf/bench.c

784

bench->report_final(state.results + env.warmup_sec,

tools/testing/selftests/bpf/bench.c

785

state.res_cnt - env.warmup_sec);

tools/testing/selftests/bpf/bench.h

69

extern struct env env;

tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c

110

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_crypto.c

71

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c

17

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

116

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

253

if (env.quiet) {

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c

255

if (env.affinity)

tools/testing/selftests/bpf/benchs/bench_bpf_loop.c

50

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_count.c

38

ctx->hits = calloc(env.producer_cnt, sizeof(*ctx->hits));

tools/testing/selftests/bpf/benchs/bench_count.c

59

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

112

nr = (env.producer_cnt + 1) / 2;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

132

nr = (env.producer_cnt + 1) / 2;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

183

bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus));

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

199

ctx.skel->bss->nr_thread = env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

294

res->hits = atomic_swap(&ctx.skel->bss->op_cnt, 0) / env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

98

if (!strcmp(use_cases[2].name, args.use_case) && env.producer_cnt % 2) {

tools/testing/selftests/bpf/benchs/bench_local_storage.c

73

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_local_storage.c

77

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

109

threads = calloc(env.producer_cnt, sizeof(*threads));

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

116

for (i = 0; i < env.producer_cnt; i++) {

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

214

creates_per_sec, creates_per_sec / env.producer_cnt);

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

238

creates_mean, creates_stddev, creates_mean / env.producer_cnt);

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c

74

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

215

if (env.quiet)

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

71

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_local_storage_rcu_tasks_trace.c

75

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

107

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

122

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

137

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

372

hits_per_prod = hits_per_sec / env.producer_cnt;

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

407

hits_mean, hits_stddev, rate, hits_mean / env.producer_cnt,

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

410

latency / lat_divisor / env.producer_cnt, unit);

tools/testing/selftests/bpf/benchs/bench_lpm_trie_map.c

87

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_rename.c

16

if (env.producer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_rename.c

20

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

112

if (args.bench_producer && strcmp(env.bench_name, "rb-libbpf")) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

122

if (args.bench_producer && env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

137

if (!args.bench_producer && env.consumer_cnt != 1) {

tools/testing/selftests/bpf/benchs/bench_ringbufs.c

142

if (args.back2back && env.producer_cnt > 1) {

tools/testing/selftests/bpf/benchs/bench_sockmap.c

238

if (env.consumer_cnt != 2 || env.producer_cnt != 1 ||

tools/testing/selftests/bpf/benchs/bench_sockmap.c

239

!env.affinity)

tools/testing/selftests/bpf/benchs/bench_strncmp.c

53

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/benchs/bench_trigger.c

149

ctx.skel->rodata->stacktrace = env.stacktrace;

tools/testing/selftests/bpf/benchs/bench_trigger.c

97

if (env.consumer_cnt != 0) {

tools/testing/selftests/bpf/jit_disasm_helpers.c

240

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c

123

!ASSERT_FALSE((env.jit_enabled && !prog_infos[i].jited_prog_len), "jited_prog_len") ||

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c

124

!ASSERT_FALSE((env.jit_enabled && !memcmp(jited_insns, zeros, sizeof(zeros))),

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c

55

if (env.verifier_stats) {

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c

66

if (env.verifier_stats)

tools/testing/selftests/bpf/prog_tests/btf.c

8307

always_log = env.verbosity > VERBOSE_NONE;

tools/testing/selftests/bpf/prog_tests/btf_module.c

15

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/btf_tag.c

60

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/build_id.c

61

if (env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/build_id.c

97

if (env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/core_reloc.c

1026

if (test_case->needs_testmod && !env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c

105

if (prog_fd < 0 || env.verbosity > VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c

77

if (btf_fd < 0 || env.verbosity > VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c

382

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c

387

if (env.verbosity > VERBOSE_NONE && exp_msg) /* we overtook log */

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c

47

if (env.jit_enabled) {

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c

64

if (env.jit_enabled) {

tools/testing/selftests/bpf/prog_tests/iters.c

307

if (env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/iters.c

75

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/ksyms_module.c

19

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/ksyms_module.c

46

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/log_buf.c

132

if (env.verbosity > VERBOSE_NONE) {

tools/testing/selftests/bpf/prog_tests/log_buf.c

91

if (env.verbosity > VERBOSE_NONE) {

tools/testing/selftests/bpf/prog_tests/log_fixup.c

131

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

160

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

66

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/log_fixup.c

97

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/prog_tests/module_fentry_shadow.c

65

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/nested_trust.c

14

if (env.has_testmod)

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

15

struct test_state *state = env.test_state;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

166

error_cnt = env.subtest_state->error_cnt;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

173

env.subtest_state->error_cnt = error_cnt;

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

175

ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, cases[i].expected, "expected output");

tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c

177

ASSERT_STREQ(env.subtest_state->log_buf, "", "expected no output");

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1333

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1345

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1362

if (env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1481

if (failed || env.verbosity >= VERBOSE_NORMAL) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1482

if (failed || env.verbosity >= VERBOSE_VERY) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1526

if (env.verbosity >= VERBOSE_NORMAL) /* this speeds up debugging */

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1530

if (err || env.verbosity >= VERBOSE_NORMAL)

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1544

fprintf(env.stderr_saved, "PROGRESS (%s): %d/%d (%.2lf%%), "

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1625

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1657

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1677

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

1710

if (env.verbosity >= VERBOSE_SUPER) {

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

744

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/reg_bounds.c

756

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/sk_assign.c

69

(env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose");

tools/testing/selftests/bpf/prog_tests/summarization.c

29

if (env.verbosity >= VERBOSE_SUPER)

tools/testing/selftests/bpf/prog_tests/summarization.c

9

if (env.verbosity >= VERBOSE_VERY)

tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c

8

if (!env.has_testmod) {

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

715

char *env;

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

717

env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC");

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

718

if (env)

tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

719

return atoi(env);

tools/testing/selftests/bpf/prog_tests/uretprobe_stack.c

92

if (env.verbosity >= VERBOSE_NORMAL) {

tools/testing/selftests/bpf/test_loader.c

1075

if (env.verbosity != VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

728

if (env.verbosity > VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

730

if (env.verbosity > VERBOSE_VERY)

tools/testing/selftests/bpf/test_loader.c

752

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

759

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

766

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

773

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

780

if (!force && env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_loader.c

921

if (env.verbosity == VERBOSE_NONE)

tools/testing/selftests/bpf/test_progs.c

1009

&env->subtest_selector.num_set,

tools/testing/selftests/bpf/test_progs.c

1010

&env->subtest_selector.num_set_len)) {

tools/testing/selftests/bpf/test_progs.c

1016

if (parse_num_list(arg, &env->test_selector.num_set,

tools/testing/selftests/bpf/test_progs.c

1017

&env->test_selector.num_set_len)) {

tools/testing/selftests/bpf/test_progs.c

1027

&env->test_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

1031

&env->test_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

1040

&env->test_selector.blacklist,

tools/testing/selftests/bpf/test_progs.c

1044

&env->test_selector.blacklist,

tools/testing/selftests/bpf/test_progs.c

1050

env->verifier_stats = true;

tools/testing/selftests/bpf/test_progs.c

1053

env->verbosity = VERBOSE_NORMAL;

tools/testing/selftests/bpf/test_progs.c

1056

env->verbosity = VERBOSE_VERY;

tools/testing/selftests/bpf/test_progs.c

1059

env->verbosity = VERBOSE_SUPER;

tools/testing/selftests/bpf/test_progs.c

1068

env_verbosity = env->verbosity;

tools/testing/selftests/bpf/test_progs.c

107

if (env.subtest_state) {

tools/testing/selftests/bpf/test_progs.c

108

if (env.subtest_state->stdout_saved)

tools/testing/selftests/bpf/test_progs.c

1081

env->get_test_cnt = true;

tools/testing/selftests/bpf/test_progs.c

1084

env->list_test_names = true;

tools/testing/selftests/bpf/test_progs.c

1088

env->workers = atoi(arg);

tools/testing/selftests/bpf/test_progs.c

1089

if (!env->workers) {

tools/testing/selftests/bpf/test_progs.c

109

fclose(env.subtest_state->stdout_saved);

tools/testing/selftests/bpf/test_progs.c

1094

env->workers = get_nprocs();

tools/testing/selftests/bpf/test_progs.c

1098

env->debug = true;

tools/testing/selftests/bpf/test_progs.c

110

env.subtest_state->stdout_saved = NULL;

tools/testing/selftests/bpf/test_progs.c

1101

env->json = fopen(arg, "w");

tools/testing/selftests/bpf/test_progs.c

1102

if (env->json == NULL) {

tools/testing/selftests/bpf/test_progs.c

111

stdout = env.test_state->stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1116

&env->tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

112

stderr = env.test_state->stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1120

&env->tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

1125

env->secs_till_kill = atoi(arg);

tools/testing/selftests/bpf/test_progs.c

1126

if (env->secs_till_kill < 0) {

tools/testing/selftests/bpf/test_progs.c

1130

if (env->secs_till_kill < env->secs_till_notify) {

tools/testing/selftests/bpf/test_progs.c

1131

env->secs_till_notify = 0;

tools/testing/selftests/bpf/test_progs.c

114

if (env.test_state->stdout_saved)

tools/testing/selftests/bpf/test_progs.c

115

fclose(env.test_state->stdout_saved);

tools/testing/selftests/bpf/test_progs.c

116

env.test_state->stdout_saved = NULL;

tools/testing/selftests/bpf/test_progs.c

117

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

118

stderr = env.stderr_saved;

tools/testing/selftests/bpf/test_progs.c

1267

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

1268

stderr = env.stderr_saved;

tools/testing/selftests/bpf/test_progs.c

1270

if (env.test) {

tools/testing/selftests/bpf/test_progs.c

1271

env.test_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

1272

dump_test_log(env.test, env.test_state, true, false, NULL);

tools/testing/selftests/bpf/test_progs.c

1287

if (env.worker_id != -1)

tools/testing/selftests/bpf/test_progs.c

1288

fprintf(stderr, "[%d]: ", env.worker_id);

tools/testing/selftests/bpf/test_progs.c

1319

for (i = 0; i < env.workers; i++)

tools/testing/selftests/bpf/test_progs.c

1320

if (env.worker_socks[i] > 0)

tools/testing/selftests/bpf/test_progs.c

1321

close(env.worker_socks[i]);

tools/testing/selftests/bpf/test_progs.c

1364

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1377

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1397

env.test = test;

tools/testing/selftests/bpf/test_progs.c

1398

env.test_state = state;

tools/testing/selftests/bpf/test_progs.c

1413

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

1420

if (verbose() && env.worker_id == -1)

tools/testing/selftests/bpf/test_progs.c

1553

env.worker_current_test[data->worker_id] = test_to_run;

tools/testing/selftests/bpf/test_progs.c

1593

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

160

struct test_env env = {};

tools/testing/selftests/bpf/test_progs.c

1602

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1610

static void calculate_summary_and_print_errors(struct test_env *env)

tools/testing/selftests/bpf/test_progs.c

1631

if (env->json) {

tools/testing/selftests/bpf/test_progs.c

1632

w = jsonw_new(env->json);

tools/testing/selftests/bpf/test_progs.c

1634

fprintf(env->stderr_saved, "Failed to create new JSON stream.");

tools/testing/selftests/bpf/test_progs.c

1673

if (env->json)

tools/testing/selftests/bpf/test_progs.c

1674

fclose(env->json);

tools/testing/selftests/bpf/test_progs.c

1679

env->succ_cnt = succ_cnt;

tools/testing/selftests/bpf/test_progs.c

1680

env->sub_succ_cnt = sub_succ_cnt;

tools/testing/selftests/bpf/test_progs.c

1681

env->fail_cnt = fail_cnt;

tools/testing/selftests/bpf/test_progs.c

1682

env->skip_cnt = skip_cnt;

tools/testing/selftests/bpf/test_progs.c

1697

dispatcher_threads = calloc(sizeof(pthread_t), env.workers);

tools/testing/selftests/bpf/test_progs.c

1698

data = calloc(sizeof(struct dispatch_data), env.workers);

tools/testing/selftests/bpf/test_progs.c

1700

env.worker_current_test = calloc(sizeof(int), env.workers);

tools/testing/selftests/bpf/test_progs.c

1701

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1705

data[i].sock_fd = env.worker_socks[i];

tools/testing/selftests/bpf/test_progs.c

1714

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1721

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1723

i, env.worker_current_test[i] + 1);

tools/testing/selftests/bpf/test_progs.c

1733

free(env.worker_current_test);

tools/testing/selftests/bpf/test_progs.c

1752

calculate_summary_and_print_errors(&env);

tools/testing/selftests/bpf/test_progs.c

1755

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

1758

pid = waitpid(env.worker_pids[i], &wstatus, 0);

tools/testing/selftests/bpf/test_progs.c

1759

if (pid != env.worker_pids[i])

tools/testing/selftests/bpf/test_progs.c

1863

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1865

env.worker_id);

tools/testing/selftests/bpf/test_progs.c

1873

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1875

env.worker_id,

tools/testing/selftests/bpf/test_progs.c

1913

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1915

env.worker_id,

tools/testing/selftests/bpf/test_progs.c

1921

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

1922

fprintf(stderr, "[%d]: unknown message.\n", env.worker_id);

tools/testing/selftests/bpf/test_progs.c

1971

env.stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

1972

env.stderr_saved = stderr;

tools/testing/selftests/bpf/test_progs.c

1974

env.secs_till_notify = 10;

tools/testing/selftests/bpf/test_progs.c

1975

env.secs_till_kill = 120;

tools/testing/selftests/bpf/test_progs.c

1976

err = argp_parse(&argp, argc, argv, 0, NULL, &env);

tools/testing/selftests/bpf/test_progs.c

198

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

1998

env.jit_enabled = is_jit_enabled();

tools/testing/selftests/bpf/test_progs.c

1999

env.nr_cpus = libbpf_num_possible_cpus();

tools/testing/selftests/bpf/test_progs.c

200

env.test->test_name, env.subtest_state->name);

tools/testing/selftests/bpf/test_progs.c

2000

if (env.nr_cpus < 0) {

tools/testing/selftests/bpf/test_progs.c

2002

env.nr_cpus);

tools/testing/selftests/bpf/test_progs.c

2006

env.has_testmod = true;

tools/testing/selftests/bpf/test_progs.c

2007

if (!env.list_test_names) {

tools/testing/selftests/bpf/test_progs.c

2012

fprintf(env.stderr_saved, "WARNING! Selftests relying on bpf_testmod.ko will be skipped.\n");

tools/testing/selftests/bpf/test_progs.c

2013

env.has_testmod = false;

tools/testing/selftests/bpf/test_progs.c

2022

test->should_run = should_run(&env.test_selector,

tools/testing/selftests/bpf/test_progs.c

203

env.test->test_name);

tools/testing/selftests/bpf/test_progs.c

2032

test->should_tmon = should_tmon(&env.tmon_selector, test->test_name);

tools/testing/selftests/bpf/test_progs.c

2036

if (env.get_test_cnt || env.list_test_names)

tools/testing/selftests/bpf/test_progs.c

2037

env.workers = 0;

tools/testing/selftests/bpf/test_progs.c

2040

env.worker_id = -1; /* main process */

tools/testing/selftests/bpf/test_progs.c

2041

if (env.workers) {

tools/testing/selftests/bpf/test_progs.c

2042

env.worker_pids = calloc(sizeof(pid_t), env.workers);

tools/testing/selftests/bpf/test_progs.c

2043

env.worker_socks = calloc(sizeof(int), env.workers);

tools/testing/selftests/bpf/test_progs.c

2044

if (env.debug)

tools/testing/selftests/bpf/test_progs.c

2045

fprintf(stdout, "Launching %d workers.\n", env.workers);

tools/testing/selftests/bpf/test_progs.c

2046

for (i = 0; i < env.workers; i++) {

tools/testing/selftests/bpf/test_progs.c

205

switch (env.watchdog_state) {

tools/testing/selftests/bpf/test_progs.c

2060

env.worker_pids[i] = pid;

tools/testing/selftests/bpf/test_progs.c

2061

env.worker_socks[i] = sv[0];

tools/testing/selftests/bpf/test_progs.c

2064

env.worker_id = i;

tools/testing/selftests/bpf/test_progs.c

2069

if (env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

207

fprintf(env.stderr_saved, "WATCHDOG: test case %s executes for %d seconds...\n",

tools/testing/selftests/bpf/test_progs.c

208

test_name, env.secs_till_notify);

tools/testing/selftests/bpf/test_progs.c

2086

if (env.get_test_cnt) {

tools/testing/selftests/bpf/test_progs.c

2087

env.succ_cnt++;

tools/testing/selftests/bpf/test_progs.c

209

timeout.it_value.tv_sec = env.secs_till_kill - env.secs_till_notify;

tools/testing/selftests/bpf/test_progs.c

2091

if (env.list_test_names) {

tools/testing/selftests/bpf/test_progs.c

2092

fprintf(env.stdout_saved, "%s\n", test->test_name);

tools/testing/selftests/bpf/test_progs.c

2093

env.succ_cnt++;

tools/testing/selftests/bpf/test_progs.c

210

env.watchdog_state = WD_KILL;

tools/testing/selftests/bpf/test_progs.c

2100

if (env.get_test_cnt) {

tools/testing/selftests/bpf/test_progs.c

2101

printf("%d\n", env.succ_cnt);

tools/testing/selftests/bpf/test_progs.c

2105

if (env.list_test_names)

tools/testing/selftests/bpf/test_progs.c

2108

calculate_summary_and_print_errors(&env);

tools/testing/selftests/bpf/test_progs.c

211

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

2110

close(env.saved_netns_fd);

tools/testing/selftests/bpf/test_progs.c

2112

if (!env.list_test_names && env.has_testmod)

tools/testing/selftests/bpf/test_progs.c

2115

free_test_selector(&env.test_selector);

tools/testing/selftests/bpf/test_progs.c

2116

free_test_selector(&env.subtest_selector);

tools/testing/selftests/bpf/test_progs.c

2117

free_test_selector(&env.tmon_selector);

tools/testing/selftests/bpf/test_progs.c

2120

if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0)

tools/testing/selftests/bpf/test_progs.c

2123

return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS;

tools/testing/selftests/bpf/test_progs.c

213

fprintf(env.stderr_saved, "Failed to arm watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

216

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

218

test_name, env.secs_till_kill);

tools/testing/selftests/bpf/test_progs.c

219

pthread_kill(env.main_thread, SIGSEGV);

tools/testing/selftests/bpf/test_progs.c

229

if (env.secs_till_kill == 0)

tools/testing/selftests/bpf/test_progs.c

231

if (env.secs_till_notify > 0) {

tools/testing/selftests/bpf/test_progs.c

232

env.watchdog_state = WD_NOTIFY;

tools/testing/selftests/bpf/test_progs.c

233

timeout.it_value.tv_sec = env.secs_till_notify;

tools/testing/selftests/bpf/test_progs.c

235

env.watchdog_state = WD_KILL;

tools/testing/selftests/bpf/test_progs.c

236

timeout.it_value.tv_sec = env.secs_till_kill;

tools/testing/selftests/bpf/test_progs.c

238

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

240

fprintf(env.stderr_saved, "Failed to start watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

248

env.watchdog_state = WD_NOTIFY;

tools/testing/selftests/bpf/test_progs.c

249

err = timer_settime(env.watchdog, 0, &timeout, NULL);

tools/testing/selftests/bpf/test_progs.c

251

fprintf(env.stderr_saved, "Failed to stop watchdog timer\n");

tools/testing/selftests/bpf/test_progs.c

262

env.main_thread = pthread_self();

tools/testing/selftests/bpf/test_progs.c

263

err = timer_create(CLOCK_MONOTONIC, &watchdog_sev, &env.watchdog);

tools/testing/selftests/bpf/test_progs.c

354

fprintf(env.stdout_saved, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name);

tools/testing/selftests/bpf/test_progs.c

356

fprintf(env.stdout_saved, "FAIL");

tools/testing/selftests/bpf/test_progs.c

358

fprintf(env.stdout_saved, "OK");

tools/testing/selftests/bpf/test_progs.c

360

fprintf(env.stdout_saved, "SKIP");

tools/testing/selftests/bpf/test_progs.c

362

fprintf(env.stdout_saved, "OK (SKIP: %d/%d)", skipped_cnt, subtests_cnt);

tools/testing/selftests/bpf/test_progs.c

364

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

370

fprintf(env.stdout_saved, "%s", log_buf);

tools/testing/selftests/bpf/test_progs.c

372

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

383

fprintf(env.stdout_saved, "#%-*s %s/%s",

tools/testing/selftests/bpf/test_progs.c

388

fprintf(env.stdout_saved, ":%s", result);

tools/testing/selftests/bpf/test_progs.c

390

fprintf(env.stdout_saved, "\n");

tools/testing/selftests/bpf/test_progs.c

421

if (env.worker_id != -1)

tools/testing/selftests/bpf/test_progs.c

48

return env.verbosity > VERBOSE_NONE;

tools/testing/selftests/bpf/test_progs.c

489

for (i = 0; i < env.nr_cpus; i++)

tools/testing/selftests/bpf/test_progs.c

506

env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY);

tools/testing/selftests/bpf/test_progs.c

507

if (env.saved_netns_fd == -1) {

tools/testing/selftests/bpf/test_progs.c

515

if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {

tools/testing/selftests/bpf/test_progs.c

523

struct prog_test_def *test = env.test;

tools/testing/selftests/bpf/test_progs.c

524

struct test_state *test_state = env.test_state;

tools/testing/selftests/bpf/test_progs.c

525

struct subtest_state *subtest_state = env.subtest_state;

tools/testing/selftests/bpf/test_progs.c

536

if (verbose() && !env.workers)

tools/testing/selftests/bpf/test_progs.c

54

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

544

env.subtest_state = NULL;

tools/testing/selftests/bpf/test_progs.c

549

struct prog_test_def *test = env.test;

tools/testing/selftests/bpf/test_progs.c

550

struct test_state *state = env.test_state;

tools/testing/selftests/bpf/test_progs.c

554

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

571

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

579

fprintf(env.stderr_saved,

tools/testing/selftests/bpf/test_progs.c

585

if (!should_run_subtest(&env.test_selector,

tools/testing/selftests/bpf/test_progs.c

586

&env.subtest_selector,

tools/testing/selftests/bpf/test_progs.c

594

subtest_state->should_tmon = match_subtest(&env.tmon_selector.whitelist,

tools/testing/selftests/bpf/test_progs.c

598

env.subtest_state = subtest_state;

tools/testing/selftests/bpf/test_progs.c

607

env.test_state->force_log = true;

tools/testing/selftests/bpf/test_progs.c

612

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

613

env.subtest_state->skipped = true;

tools/testing/selftests/bpf/test_progs.c

615

env.test_state->skip_cnt++;

tools/testing/selftests/bpf/test_progs.c

620

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

621

env.subtest_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

623

env.test_state->error_cnt++;

tools/testing/selftests/bpf/test_progs.c

630

if (!env.test->need_cgroup_cleanup) {

tools/testing/selftests/bpf/test_progs.c

634

env.test->test_num, env.test->test_name);

tools/testing/selftests/bpf/test_progs.c

638

env.test->need_cgroup_cleanup = true;

tools/testing/selftests/bpf/test_progs.c

64

stdout = env.stdout_saved;

tools/testing/selftests/bpf/test_progs.c

645

env.test->test_num, env.test->test_name, path, errno);

tools/testing/selftests/bpf/test_progs.c

652

env.test->test_num, env.test->test_name, path, errno);

tools/testing/selftests/bpf/test_progs.c

69

if (env.subtest_state)

tools/testing/selftests/bpf/test_progs.c

70

env.subtest_state->stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

72

env.test_state->stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

777

if (env.test->should_tmon ||

tools/testing/selftests/bpf/test_progs.c

778

(env.subtest_state && env.subtest_state->should_tmon)) {

tools/testing/selftests/bpf/test_progs.c

779

test_name = env.test->test_name;

tools/testing/selftests/bpf/test_progs.c

780

subtest_name = env.subtest_state ? env.subtest_state->name : NULL;

tools/testing/selftests/bpf/test_progs.c

81

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

86

env.stdout_saved = stdout;

tools/testing/selftests/bpf/test_progs.c

87

env.stderr_saved = stderr;

tools/testing/selftests/bpf/test_progs.c

963

if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)

tools/testing/selftests/bpf/test_progs.c

98

if (verbose() && env.worker_id == -1) {

tools/testing/selftests/bpf/test_progs.c

999

struct test_env *env = state->input;

tools/testing/selftests/bpf/test_progs.h

181

extern struct test_env env;

tools/testing/selftests/bpf/test_sockmap.c

139

struct test_env env;

tools/testing/selftests/bpf/test_sockmap.c

165

env.subtest_num++;

tools/testing/selftests/bpf/test_sockmap.c

170

env.fail_cnt++;

tools/testing/selftests/bpf/test_sockmap.c

175

env.succ_cnt++;

tools/testing/selftests/bpf/test_sockmap.c

193

env.type = o->map;

tools/testing/selftests/bpf/test_sockmap.c

194

env.subtest = t->title;

tools/testing/selftests/bpf/test_sockmap.c

195

env.prepend = o->prepend;

tools/testing/selftests/bpf/test_sockmap.c

196

env.test_num++;

tools/testing/selftests/bpf/test_sockmap.c

197

env.subtest_num = 0;

tools/testing/selftests/bpf/test_sockmap.c

198

env.fail_last = env.fail_cnt;

tools/testing/selftests/bpf/test_sockmap.c

205

int error = env.fail_cnt - env.fail_last;

tools/testing/selftests/bpf/test_sockmap.c

206

int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);

tools/testing/selftests/bpf/test_sockmap.c

212

env.test_num, env.subtest_num,

tools/testing/selftests/bpf/test_sockmap.c

214

env.prepend ? : "",

tools/testing/selftests/bpf/test_sockmap.c

215

env.subtest, error ? "FAIL" : "OK");

tools/testing/selftests/bpf/test_sockmap.c

221

env.succ_cnt, env.fail_cnt);

tools/testing/selftests/bpf/veristat.c

1120

for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {

tools/testing/selftests/bpf/veristat.c

1305

if (!env.quiet) {

tools/testing/selftests/bpf/veristat.c

1311

if (!env.quiet) {

tools/testing/selftests/bpf/veristat.c

1354

for (i = 0; i < env.output_spec.spec_cnt; i++)

tools/testing/selftests/bpf/veristat.c

1355

if (env.output_spec.ids[i] == id)

tools/testing/selftests/bpf/veristat.c

1418

close(env.memory_peak_fd);

tools/testing/selftests/bpf/veristat.c

1420

if (env.orig_cgroup[0]) {

tools/testing/selftests/bpf/veristat.c

1421

snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.orig_cgroup);

tools/testing/selftests/bpf/veristat.c

1424

log_errno("moving self to original cgroup %s\n", env.orig_cgroup);

tools/testing/selftests/bpf/veristat.c

1427

if (env.stat_cgroup[0]) {

tools/testing/selftests/bpf/veristat.c

1428

err = rmdir(env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1430

log_errno("deletion of cgroup %s", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1433

env.memory_peak_fd = -1;

tools/testing/selftests/bpf/veristat.c

1434

env.orig_cgroup[0] = 0;

tools/testing/selftests/bpf/veristat.c

1435

env.stat_cgroup[0] = 0;

tools/testing/selftests/bpf/veristat.c

1448

env.memory_peak_fd = -1;

tools/testing/selftests/bpf/veristat.c

1458

else if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1468

else if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1473

snprintf(env.orig_cgroup, sizeof(env.orig_cgroup), "%s/%s", cgroup_fs_mount, buf);

tools/testing/selftests/bpf/veristat.c

1481

strcpy(env.stat_cgroup, buf);

tools/testing/selftests/bpf/veristat.c

1483

snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1490

snprintf(buf, sizeof(buf), "%s/memory.peak", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1491

env.memory_peak_fd = open(buf, O_RDWR | O_APPEND);

tools/testing/selftests/bpf/veristat.c

1492

if (env.memory_peak_fd < 0) {

tools/testing/selftests/bpf/veristat.c

1500

if (!env.quiet)

tools/testing/selftests/bpf/veristat.c

1511

if (env.memory_peak_fd < 0)

tools/testing/selftests/bpf/veristat.c

1514

err = pread(env.memory_peak_fd, buf, sizeof(buf) - 1, 0);

tools/testing/selftests/bpf/veristat.c

1516

log_errno("pread(%s/memory.peak)", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1524

log_errno("%s/memory.peak:strtoll(%s)", env.stat_cgroup, buf);

tools/testing/selftests/bpf/veristat.c

1536

if (env.memory_peak_fd < 0)

tools/testing/selftests/bpf/veristat.c

1539

err = pwrite(env.memory_peak_fd, buf, sizeof(buf), 0);

tools/testing/selftests/bpf/veristat.c

1541

log_errno("pwrite(%s/memory.peak)", env.stat_cgroup);

tools/testing/selftests/bpf/veristat.c

1621

env.progs_skipped++;

tools/testing/selftests/bpf/veristat.c

1625

tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));

tools/testing/selftests/bpf/veristat.c

1628

env.prog_stats = tmp;

tools/testing/selftests/bpf/veristat.c

1629

stats = &env.prog_stats[env.prog_stat_cnt++];

tools/testing/selftests/bpf/veristat.c

1632

if (env.verbose || env.top_src_lines > 0) {

tools/testing/selftests/bpf/veristat.c

1633

buf_sz = env.log_size ? env.log_size : max_verifier_log_size();

tools/testing/selftests/bpf/veristat.c

1638

log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);

tools/testing/selftests/bpf/veristat.c

1640

if (env.top_src_lines > 0 && env.log_level == 0)

tools/testing/selftests/bpf/veristat.c

1646

log_level = 4 | (env.log_fixed ? 8 : 0);

tools/testing/selftests/bpf/veristat.c

1656

if (env.force_checkpoints)

tools/testing/selftests/bpf/veristat.c

1658

if (env.force_reg_invariants)

tools/testing/selftests/bpf/veristat.c

1670

env.progs_processed++;

tools/testing/selftests/bpf/veristat.c

1684

if (env.dump_mode & DUMP_JITED)

tools/testing/selftests/bpf/veristat.c

1686

if (env.dump_mode & DUMP_XLATED)

tools/testing/selftests/bpf/veristat.c

1692

if (env.verbose) {

tools/testing/selftests/bpf/veristat.c

1697

if (env.top_src_lines > 0)

tools/testing/selftests/bpf/veristat.c

1820

err = append_var_preset(&env.presets, &env.npresets, buf);

tools/testing/selftests/bpf/veristat.c

2192

if (env.verbose)

tools/testing/selftests/bpf/veristat.c

2194

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2198

if (env.verbose)

tools/testing/selftests/bpf/veristat.c

2200

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2204

if (!env.quiet && env.out_fmt == RESFMT_TABLE)

tools/testing/selftests/bpf/veristat.c

2217

env.files_skipped++;

tools/testing/selftests/bpf/veristat.c

2222

env.files_processed++;

tools/testing/selftests/bpf/veristat.c

2231

err = set_global_vars(obj, env.presets, env.npresets);

tools/testing/selftests/bpf/veristat.c

2250

err = set_global_vars(tobj, env.presets, env.npresets);

tools/testing/selftests/bpf/veristat.c

2328

for (i = 0; i < env.sort_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2329

cmp = cmp_stat(s1, s2, env.sort_spec.ids[i],

tools/testing/selftests/bpf/veristat.c

2330

env.sort_spec.asc[i], env.sort_spec.abs[i]);

tools/testing/selftests/bpf/veristat.c

237

} env;

tools/testing/selftests/bpf/veristat.c

241

if (!env.verbose)

tools/testing/selftests/bpf/veristat.c

2427

for (i = 0; i < env.sort_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2429

env.sort_spec.ids[i],

tools/testing/selftests/bpf/veristat.c

243

if (level == LIBBPF_DEBUG && !env.debug)

tools/testing/selftests/bpf/veristat.c

2430

env.sort_spec.variants[i],

tools/testing/selftests/bpf/veristat.c

2431

env.sort_spec.asc[i],

tools/testing/selftests/bpf/veristat.c

2432

env.sort_spec.abs[i]);

tools/testing/selftests/bpf/veristat.c

2451

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2452

len = env.output_spec.lens[i];

tools/testing/selftests/bpf/veristat.c

2466

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2467

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2468

int *max_len = &env.output_spec.lens[i];

tools/testing/selftests/bpf/veristat.c

2479

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2484

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2544

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2545

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2546

int *max_len = &env.output_spec.lens[i], len;

tools/testing/selftests/bpf/veristat.c

2566

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2574

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2583

env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped);

tools/testing/selftests/bpf/veristat.c

2787

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2788

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2792

int len = env.output_spec.lens[3 * i + j];

tools/testing/selftests/bpf/veristat.c

2809

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2810

int id = env.output_spec.ids[i];

tools/testing/selftests/bpf/veristat.c

2815

int *max_len = &env.output_spec.lens[3 * i + j];

tools/testing/selftests/bpf/veristat.c

2816

bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);

tools/testing/selftests/bpf/veristat.c

2855

for (i = 0; i < env.output_spec.spec_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2856

int id = env.output_spec.ids[i], len;

tools/testing/selftests/bpf/veristat.c

2857

int *max_len_base = &env.output_spec.lens[3 * i + 0];

tools/testing/selftests/bpf/veristat.c

2858

int *max_len_comp = &env.output_spec.lens[3 * i + 1];

tools/testing/selftests/bpf/veristat.c

2859

int *max_len_diff = &env.output_spec.lens[3 * i + 2];

tools/testing/selftests/bpf/veristat.c

2933

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2943

if (i == env.output_spec.spec_cnt - 1)

tools/testing/selftests/bpf/veristat.c

2992

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

2993

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

3001

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3002

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

3022

if (env.filename_cnt != 2) {

tools/testing/selftests/bpf/veristat.c

3028

err = parse_stats_csv(env.filenames[0], &base_specs,

tools/testing/selftests/bpf/veristat.c

3029

&env.baseline_stats, &env.baseline_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3031

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);

tools/testing/selftests/bpf/veristat.c

3034

err = parse_stats_csv(env.filenames[1], &comp_specs,

tools/testing/selftests/bpf/veristat.c

3035

&env.prog_stats, &env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3037

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);

tools/testing/selftests/bpf/veristat.c

3047

env.filenames[0], env.filenames[1],

tools/testing/selftests/bpf/veristat.c

3054

env.filenames[0], env.filenames[1],

tools/testing/selftests/bpf/veristat.c

3065

tmp_sort_spec = env.sort_spec;

tools/testing/selftests/bpf/veristat.c

3066

env.sort_spec = join_sort_spec;

tools/testing/selftests/bpf/veristat.c

3067

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3068

qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3069

env.sort_spec = tmp_sort_spec;

tools/testing/selftests/bpf/veristat.c

3077

while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {

tools/testing/selftests/bpf/veristat.c

3083

base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;

tools/testing/selftests/bpf/veristat.c

3084

comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;

tools/testing/selftests/bpf/veristat.c

3088

i, env.filenames[0]);

tools/testing/selftests/bpf/veristat.c

3093

j, env.filenames[1]);

tools/testing/selftests/bpf/veristat.c

3097

tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats));

tools/testing/selftests/bpf/veristat.c

3100

env.join_stats = tmp;

tools/testing/selftests/bpf/veristat.c

3102

join = &env.join_stats[env.join_stat_cnt];

tools/testing/selftests/bpf/veristat.c

3130

env.join_stat_cnt += 1;

tools/testing/selftests/bpf/veristat.c

3134

qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats);

tools/testing/selftests/bpf/veristat.c

3141

if (env.out_fmt == RESFMT_TABLE)

tools/testing/selftests/bpf/veristat.c

3144

cur_fmt = env.out_fmt;

tools/testing/selftests/bpf/veristat.c

3151

for (i = 0; i < env.join_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3152

const struct verif_stats_join *join = &env.join_stats[i];

tools/testing/selftests/bpf/veristat.c

3157

if (env.top_n && cnt >= env.top_n)

tools/testing/selftests/bpf/veristat.c

3201

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3202

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

3210

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3211

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

3229

if (env.out_fmt == RESFMT_TABLE) {

tools/testing/selftests/bpf/veristat.c

3232

for (i = 0; i < env.prog_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3233

stats = &env.prog_stats[i];

tools/testing/selftests/bpf/veristat.c

3242

output_headers(env.out_fmt);

tools/testing/selftests/bpf/veristat.c

3243

for (i = 0; i < env.prog_stat_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3244

stats = &env.prog_stats[i];

tools/testing/selftests/bpf/veristat.c

3247

if (env.top_n && cnt >= env.top_n)

tools/testing/selftests/bpf/veristat.c

3249

output_stats(stats, env.out_fmt, i == last_stat_idx);

tools/testing/selftests/bpf/veristat.c

3258

if (env.filename_cnt == 0) {

tools/testing/selftests/bpf/veristat.c

3265

for (i = 0; i < env.filename_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3266

err = process_obj(env.filenames[i]);

tools/testing/selftests/bpf/veristat.c

3268

fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);

tools/testing/selftests/bpf/veristat.c

327

env.show_version = true;

tools/testing/selftests/bpf/veristat.c

3273

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3287

if (env.filename_cnt != 1) {

tools/testing/selftests/bpf/veristat.c

3293

err = parse_stats_csv(env.filenames[0], &specs,

tools/testing/selftests/bpf/veristat.c

3294

&env.prog_stats, &env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3296

fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);

tools/testing/selftests/bpf/veristat.c

330

env.verbose = true;

tools/testing/selftests/bpf/veristat.c

3300

qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);

tools/testing/selftests/bpf/veristat.c

3314

if (env.show_version) {

tools/testing/selftests/bpf/veristat.c

3319

if (env.verbose && env.quiet) {

tools/testing/selftests/bpf/veristat.c

3324

if (env.verbose && env.log_level == 0)

tools/testing/selftests/bpf/veristat.c

3325

env.log_level = 1;

tools/testing/selftests/bpf/veristat.c

3327

if (env.output_spec.spec_cnt == 0) {

tools/testing/selftests/bpf/veristat.c

3328

if (env.out_fmt == RESFMT_CSV)

tools/testing/selftests/bpf/veristat.c

3329

env.output_spec = default_csv_output_spec;

tools/testing/selftests/bpf/veristat.c

333

env.debug = true;

tools/testing/selftests/bpf/veristat.c

3331

env.output_spec = default_output_spec;

tools/testing/selftests/bpf/veristat.c

3333

if (env.sort_spec.spec_cnt == 0)

tools/testing/selftests/bpf/veristat.c

3334

env.sort_spec = default_sort_spec;

tools/testing/selftests/bpf/veristat.c

3336

if (env.comparison_mode && env.replay_mode) {

tools/testing/selftests/bpf/veristat.c

334

env.verbose = true;

tools/testing/selftests/bpf/veristat.c

3342

if (env.comparison_mode)

tools/testing/selftests/bpf/veristat.c

3344

else if (env.replay_mode)

tools/testing/selftests/bpf/veristat.c

3349

free_verif_stats(env.prog_stats, env.prog_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3350

free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);

tools/testing/selftests/bpf/veristat.c

3351

free(env.join_stats);

tools/testing/selftests/bpf/veristat.c

3352

for (i = 0; i < env.filename_cnt; i++)

tools/testing/selftests/bpf/veristat.c

3353

free(env.filenames[i]);

tools/testing/selftests/bpf/veristat.c

3354

free(env.filenames);

tools/testing/selftests/bpf/veristat.c

3355

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3356

free(env.allow_filters[i].any_glob);

tools/testing/selftests/bpf/veristat.c

3357

free(env.allow_filters[i].file_glob);

tools/testing/selftests/bpf/veristat.c

3358

free(env.allow_filters[i].prog_glob);

tools/testing/selftests/bpf/veristat.c

3360

free(env.allow_filters);

tools/testing/selftests/bpf/veristat.c

3361

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

3362

free(env.deny_filters[i].any_glob);

tools/testing/selftests/bpf/veristat.c

3363

free(env.deny_filters[i].file_glob);

tools/testing/selftests/bpf/veristat.c

3364

free(env.deny_filters[i].prog_glob);

tools/testing/selftests/bpf/veristat.c

3366

free(env.deny_filters);

tools/testing/selftests/bpf/veristat.c

3367

for (i = 0; i < env.npresets; ++i) {

tools/testing/selftests/bpf/veristat.c

3368

free(env.presets[i].full_name);

tools/testing/selftests/bpf/veristat.c

3369

for (j = 0; j < env.presets[i].atom_count; ++j) {

tools/testing/selftests/bpf/veristat.c

337

env.quiet = true;

tools/testing/selftests/bpf/veristat.c

3370

switch (env.presets[i].atoms[j].type) {

tools/testing/selftests/bpf/veristat.c

3372

free(env.presets[i].atoms[j].name);

tools/testing/selftests/bpf/veristat.c

3375

if (env.presets[i].atoms[j].index.type == ENUMERATOR)

tools/testing/selftests/bpf/veristat.c

3376

free(env.presets[i].atoms[j].index.svalue);

tools/testing/selftests/bpf/veristat.c

3380

free(env.presets[i].atoms);

tools/testing/selftests/bpf/veristat.c

3381

if (env.presets[i].value.type == ENUMERATOR)

tools/testing/selftests/bpf/veristat.c

3382

free(env.presets[i].value.svalue);

tools/testing/selftests/bpf/veristat.c

3384

free(env.presets);

tools/testing/selftests/bpf/veristat.c

340

err = parse_stats(arg, &env.output_spec);

tools/testing/selftests/bpf/veristat.c

345

err = parse_stats(arg, &env.sort_spec);

tools/testing/selftests/bpf/veristat.c

351

env.out_fmt = RESFMT_TABLE;

tools/testing/selftests/bpf/veristat.c

353

env.out_fmt = RESFMT_CSV;

tools/testing/selftests/bpf/veristat.c

361

env.log_level = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

368

env.log_fixed = true;

tools/testing/selftests/bpf/veristat.c

372

env.log_size = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

379

env.force_checkpoints = true;

tools/testing/selftests/bpf/veristat.c

382

env.force_reg_invariants = true;

tools/testing/selftests/bpf/veristat.c

386

env.top_n = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

393

env.comparison_mode = true;

tools/testing/selftests/bpf/veristat.c

396

env.replay_mode = true;

tools/testing/selftests/bpf/veristat.c

402

err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);

tools/testing/selftests/bpf/veristat.c

404

err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);

tools/testing/selftests/bpf/veristat.c

412

env.top_src_lines = strtol(arg, NULL, 10);

tools/testing/selftests/bpf/veristat.c

422

err = append_var_preset(&env.presets, &env.npresets, arg);

tools/testing/selftests/bpf/veristat.c

441

env.dump_mode |= DUMP_XLATED;

tools/testing/selftests/bpf/veristat.c

443

env.dump_mode |= DUMP_JITED;

tools/testing/selftests/bpf/veristat.c

521

for (i = 0; i < env.deny_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

522

f = &env.deny_filters[i];

tools/testing/selftests/bpf/veristat.c

536

for (i = 0; i < env.allow_filter_cnt; i++) {

tools/testing/selftests/bpf/veristat.c

537

f = &env.allow_filters[i];

tools/testing/selftests/bpf/veristat.c

728

err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);

tools/testing/selftests/bpf/veristat.c

730

err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);

tools/testing/selftests/bpf/veristat.c

752

tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));

tools/testing/selftests/bpf/veristat.c

755

env.filenames = tmp;

tools/testing/selftests/bpf/veristat.c

756

env.filenames[env.filename_cnt] = strdup(path);

tools/testing/selftests/bpf/veristat.c

757

if (!env.filenames[env.filename_cnt])

tools/testing/selftests/bpf/veristat.c

759

env.filename_cnt++;

tools/testing/selftests/bpf/xdp_features.c

100

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

101

env.feature.action = XDP_TX;

tools/testing/selftests/bpf/xdp_features.c

103

env.feature.drv_feature = NETDEV_XDP_ACT_REDIRECT;

tools/testing/selftests/bpf/xdp_features.c

104

env.feature.action = XDP_REDIRECT;

tools/testing/selftests/bpf/xdp_features.c

106

env.feature.drv_feature = NETDEV_XDP_ACT_NDO_XMIT;

tools/testing/selftests/bpf/xdp_features.c

116

switch (env.feature.action) {

tools/testing/selftests/bpf/xdp_features.c

131

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT)

tools/testing/selftests/bpf/xdp_features.c

141

env.verbosity = true;

tools/testing/selftests/bpf/xdp_features.c

144

env.is_tester = true;

tools/testing/selftests/bpf/xdp_features.c

155

&env.dut_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

164

&env.dut_ctrl_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

172

if (make_sockaddr(AF_INET6, arg, 0, &env.tester_addr, NULL)) {

tools/testing/selftests/bpf/xdp_features.c

187

env.ifindex = if_nametoindex(arg);

tools/testing/selftests/bpf/xdp_features.c

188

if (!env.ifindex)

tools/testing/selftests/bpf/xdp_features.c

189

env.ifindex = strtoul(arg, NULL, 0);

tools/testing/selftests/bpf/xdp_features.c

190

if (!env.ifindex || !if_indextoname(env.ifindex, env.ifname)) {

tools/testing/selftests/bpf/xdp_features.c

213

env.feature.drv_feature = NETDEV_XDP_ACT_NDO_XMIT;

tools/testing/selftests/bpf/xdp_features.c

214

env.feature.action = -EINVAL;

tools/testing/selftests/bpf/xdp_features.c

215

env.ifindex = -ENODEV;

tools/testing/selftests/bpf/xdp_features.c

216

strscpy(env.ifname, "unknown");

tools/testing/selftests/bpf/xdp_features.c

218

&env.dut_ctrl_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

220

&env.dut_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

221

make_sockaddr(AF_INET6, "::ffff:127.0.0.1", 0, &env.tester_addr, NULL);

tools/testing/selftests/bpf/xdp_features.c

262

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

271

env.ifname, strerror(-err));

tools/testing/selftests/bpf/xdp_features.c

281

enum xdp_action action = env.feature.action;

tools/testing/selftests/bpf/xdp_features.c

286

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT) {

tools/testing/selftests/bpf/xdp_features.c

288

.ifindex = env.ifindex,

tools/testing/selftests/bpf/xdp_features.c

333

err = bpf_xdp_attach(env.ifindex, bpf_program__fd(prog), flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

336

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

373

"Failed creating control socket on device %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

381

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

39

} env;

tools/testing/selftests/bpf/xdp_features.c

424

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

435

err = bpf_xdp_query(env.ifindex, XDP_FLAGS_DRV_MODE,

tools/testing/selftests/bpf/xdp_features.c

440

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

48

if (level == LIBBPF_DEBUG && !env.verbosity)

tools/testing/selftests/bpf/xdp_features.c

489

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

511

switch (env.feature.action) {

tools/testing/selftests/bpf/xdp_features.c

523

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT)

tools/testing/selftests/bpf/xdp_features.c

560

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

568

(struct sockaddr *)&env.dut_addr, sizeof(env.dut_addr));

tools/testing/selftests/bpf/xdp_features.c

593

err = connect(sockfd, (struct sockaddr *)&env.dut_ctrl_addr,

tools/testing/selftests/bpf/xdp_features.c

594

sizeof(env.dut_ctrl_addr));

tools/testing/selftests/bpf/xdp_features.c

610

if (env.feature.drv_feature == NETDEV_XDP_ACT_NDO_XMIT ||

tools/testing/selftests/bpf/xdp_features.c

611

env.feature.action == XDP_TX)

tools/testing/selftests/bpf/xdp_features.c

616

err = bpf_xdp_attach(env.ifindex, bpf_program__fd(prog), flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

619

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

648

env.feature.drv_feature & advertised_feature ? GREEN("ADVERTISED")

tools/testing/selftests/bpf/xdp_features.c

651

bpf_xdp_detach(env.ifindex, flags, NULL);

tools/testing/selftests/bpf/xdp_features.c

674

if (env.ifindex < 0) {

tools/testing/selftests/bpf/xdp_features.c

675

fprintf(stderr, "Invalid device name %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

687

((struct sockaddr_in6 *)&env.tester_addr)->sin6_addr;

tools/testing/selftests/bpf/xdp_features.c

689

((struct sockaddr_in6 *)&env.dut_addr)->sin6_addr;

tools/testing/selftests/bpf/xdp_features.c

704

if (env.is_tester) {

tools/testing/selftests/bpf/xdp_features.c

707

env.ifname);

tools/testing/selftests/bpf/xdp_features.c

711

fprintf(stdout, "Starting test on device %s\n", env.ifname);

tools/testing/selftests/bpf/xdp_features.c

91

env.feature.action = XDP_PASS;

tools/testing/selftests/bpf/xdp_features.c

92

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

94

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

95

env.feature.action = XDP_DROP;

tools/testing/selftests/bpf/xdp_features.c

97

env.feature.drv_feature = NETDEV_XDP_ACT_BASIC;

tools/testing/selftests/bpf/xdp_features.c

98

env.feature.action = XDP_ABORTED;

tools/testing/selftests/mm/mkdirty.c

32

static sigjmp_buf env;

tools/testing/selftests/mm/mkdirty.c

37

siglongjmp(env, 1);

tools/testing/selftests/mm/mkdirty.c

38

siglongjmp(env, 2);

tools/testing/selftests/mm/mkdirty.c

51

ret = sigsetjmp(env, 1);

tools/testing/selftests/powerpc/mm/wild_bctr.c

113

unsigned long env;

virt/kvm/kvm_main.c

6297

struct kobj_uevent_env *env;

virt/kvm/kvm_main.c

6314

env = kzalloc_obj(*env);

virt/kvm/kvm_main.c

6315

if (!env)

virt/kvm/kvm_main.c

6318

add_uevent_var(env, "CREATED=%llu", created);

virt/kvm/kvm_main.c

6319

add_uevent_var(env, "COUNT=%llu", active);

virt/kvm/kvm_main.c

6322

add_uevent_var(env, "EVENT=create");

virt/kvm/kvm_main.c

6325

add_uevent_var(env, "EVENT=destroy");

virt/kvm/kvm_main.c

6327

add_uevent_var(env, "PID=%d", kvm->userspace_pid);

virt/kvm/kvm_main.c

6335

add_uevent_var(env, "STATS_PATH=%s", tmp);

virt/kvm/kvm_main.c

6340

env->envp[env->envp_idx++] = NULL;

virt/kvm/kvm_main.c

6341

kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);

virt/kvm/kvm_main.c

6342

kfree(env);