arch/x86/kernel/cpu/common.c

2352

unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));

arch/x86/kernel/cpu/common.c

2356

wrmsrq(MSR_TSC_AUX, cpudata);

arch/x86/kernel/cpu/common.c

2359

d.limit0 = cpudata;

arch/x86/kernel/cpu/common.c

2360

d.limit1 = cpudata >> 16;

drivers/cpufreq/amd-pstate-ut.c

154

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate-ut.c

159

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate-ut.c

185

cur_perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate-ut.c

186

if (highest_perf != cur_perf.highest_perf && !cpudata->hw_prefcore) {

drivers/cpufreq/amd-pstate-ut.c

226

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate-ut.c

231

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate-ut.c

233

if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) &&

drivers/cpufreq/amd-pstate-ut.c

234

(cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) &&

drivers/cpufreq/amd-pstate-ut.c

235

(cpudata->lowest_nonlinear_freq >= policy->cpuinfo.min_freq) &&

drivers/cpufreq/amd-pstate-ut.c

238

__func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq,

drivers/cpufreq/amd-pstate-ut.c

239

cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq);

drivers/cpufreq/amd-pstate-ut.c

243

if (cpudata->lowest_nonlinear_freq != policy->min) {

drivers/cpufreq/amd-pstate-ut.c

245

__func__, cpu, cpudata->lowest_nonlinear_freq, policy->min);

drivers/cpufreq/amd-pstate-ut.c

249

if (cpudata->boost_supported) {

drivers/cpufreq/amd-pstate-ut.c

251

(policy->max != cpudata->nominal_freq)) {

drivers/cpufreq/amd-pstate-ut.c

254

cpudata->nominal_freq);

drivers/cpufreq/amd-pstate-ut.c

286

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate-ut.c

297

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate-ut.c

299

orig_dynamic_epp = cpudata->dynamic_epp;

drivers/cpufreq/amd-pstate-ut.c

320

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate-ut.c

321

orig_policy = cpudata->policy;

drivers/cpufreq/amd-pstate-ut.c

322

cpudata->policy = CPUFREQ_POLICY_POWERSAVE;

drivers/cpufreq/amd-pstate-ut.c

329

if (cpudata->dynamic_epp) {

drivers/cpufreq/amd-pstate-ut.c

382

cpudata->policy = orig_policy;

drivers/cpufreq/amd-pstate-ut.c

492

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate-ut.c

497

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate-ut.c

498

has_prefcore = cpudata->hw_prefcore;

drivers/cpufreq/amd-pstate.c

1008

WRITE_ONCE(cpudata->perf, perf);

drivers/cpufreq/amd-pstate.c

1014

WRITE_ONCE(cpudata->nominal_freq, nominal_freq);

drivers/cpufreq/amd-pstate.c

1018

WRITE_ONCE(cpudata->max_freq, max_freq);

drivers/cpufreq/amd-pstate.c

1021

WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq);

drivers/cpufreq/amd-pstate.c

1045

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

1059

cpudata = kzalloc_obj(*cpudata);

drivers/cpufreq/amd-pstate.c

1060

if (!cpudata)

drivers/cpufreq/amd-pstate.c

1063

cpudata->cpu = policy->cpu;

drivers/cpufreq/amd-pstate.c

1065

ret = amd_pstate_init_perf(cpudata);

drivers/cpufreq/amd-pstate.c

1069

amd_pstate_init_prefcore(cpudata);

drivers/cpufreq/amd-pstate.c

1071

ret = amd_pstate_init_freq(cpudata);

drivers/cpufreq/amd-pstate.c

1075

ret = amd_pstate_init_boost_support(cpudata);

drivers/cpufreq/amd-pstate.c

1082

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1084

policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq,

drivers/cpufreq/amd-pstate.c

1086

policy->cpuinfo.max_freq = cpudata->max_freq;

drivers/cpufreq/amd-pstate.c

1088

policy->driver_data = cpudata;

drivers/cpufreq/amd-pstate.c

1093

policy->boost_supported = READ_ONCE(cpudata->boost_supported);

drivers/cpufreq/amd-pstate.c

1107

ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],

drivers/cpufreq/amd-pstate.c

1114

ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],

drivers/cpufreq/amd-pstate.c

1128

freq_qos_remove_request(&cpudata->req[0]);

drivers/cpufreq/amd-pstate.c

1131

kfree(cpudata);

drivers/cpufreq/amd-pstate.c

1138

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1139

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1143

amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);

drivers/cpufreq/amd-pstate.c

1145

freq_qos_remove_request(&cpudata->req[1]);

drivers/cpufreq/amd-pstate.c

1146

freq_qos_remove_request(&cpudata->req[0]);

drivers/cpufreq/amd-pstate.c

1148

kfree(cpudata);

drivers/cpufreq/amd-pstate.c

1153

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1156

return cpudata->epp_default_ac;

drivers/cpufreq/amd-pstate.c

1158

return cpudata->epp_default_dc;

drivers/cpufreq/amd-pstate.c

1164

struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb);

drivers/cpufreq/amd-pstate.c

1165

struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu);

drivers/cpufreq/amd-pstate.c

1173

if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED)

drivers/cpufreq/amd-pstate.c

1180

pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret);

drivers/cpufreq/amd-pstate.c

1197

struct amd_cpudata *cpudata = dev_get_drvdata(dev);

drivers/cpufreq/amd-pstate.c

1199

*profile = cpudata->current_profile;

drivers/cpufreq/amd-pstate.c

1207

struct amd_cpudata *cpudata = dev_get_drvdata(dev);

drivers/cpufreq/amd-pstate.c

1208

struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu);

drivers/cpufreq/amd-pstate.c

1233

cpudata->current_profile = profile;

drivers/cpufreq/amd-pstate.c

1246

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1248

if (cpudata->power_nb.notifier_call)

drivers/cpufreq/amd-pstate.c

1249

power_supply_unreg_notifier(&cpudata->power_nb);

drivers/cpufreq/amd-pstate.c

1250

if (cpudata->ppdev) {

drivers/cpufreq/amd-pstate.c

1251

platform_profile_remove(cpudata->ppdev);

drivers/cpufreq/amd-pstate.c

1252

cpudata->ppdev = NULL;

drivers/cpufreq/amd-pstate.c

1254

kfree(cpudata->profile_name);

drivers/cpufreq/amd-pstate.c

1255

cpudata->dynamic_epp = false;

drivers/cpufreq/amd-pstate.c

1261

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1265

switch (cpudata->current_profile) {

drivers/cpufreq/amd-pstate.c

1276

pr_err("Unknown Platform Profile %d\n", cpudata->current_profile);

drivers/cpufreq/amd-pstate.c

1283

cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu);

drivers/cpufreq/amd-pstate.c

1284

if (!cpudata->profile_name)

drivers/cpufreq/amd-pstate.c

1287

cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu),

drivers/cpufreq/amd-pstate.c

1288

cpudata->profile_name,

drivers/cpufreq/amd-pstate.c

1291

if (IS_ERR(cpudata->ppdev)) {

drivers/cpufreq/amd-pstate.c

1292

ret = PTR_ERR(cpudata->ppdev);

drivers/cpufreq/amd-pstate.c

1297

if (cpudata->epp_default_ac != cpudata->epp_default_dc) {

drivers/cpufreq/amd-pstate.c

1298

cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier;

drivers/cpufreq/amd-pstate.c

1299

ret = power_supply_reg_notifier(&cpudata->power_nb);

drivers/cpufreq/amd-pstate.c

1304

cpudata->dynamic_epp = true;

drivers/cpufreq/amd-pstate.c

1324

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1326

return sysfs_emit(buf, "%u\n", cpudata->max_freq);

drivers/cpufreq/amd-pstate.c

1332

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

1335

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1336

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1339

perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf));

drivers/cpufreq/amd-pstate.c

1349

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

1351

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1353

return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf);

drivers/cpufreq/amd-pstate.c

1360

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1362

perf = READ_ONCE(cpudata->prefcore_ranking);

drivers/cpufreq/amd-pstate.c

1371

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1373

hw_prefcore = READ_ONCE(cpudata->hw_prefcore);

drivers/cpufreq/amd-pstate.c

1382

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1384

if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)

drivers/cpufreq/amd-pstate.c

1399

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1404

if (cpudata->dynamic_epp) {

drivers/cpufreq/amd-pstate.c

1422

epp = cpudata->epp_default_dc;

drivers/cpufreq/amd-pstate.c

1425

if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {

drivers/cpufreq/amd-pstate.c

1434

cpudata->raw_epp = raw_epp;

drivers/cpufreq/amd-pstate.c

1442

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1445

epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

1447

if (cpudata->raw_epp)

drivers/cpufreq/amd-pstate.c

1474

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1475

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1487

floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq);

drivers/cpufreq/amd-pstate.c

1491

cpudata->floor_freq = freq;

drivers/cpufreq/amd-pstate.c

1498

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1500

return sysfs_emit(buf, "%u\n", cpudata->floor_freq);

drivers/cpufreq/amd-pstate.c

1505

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1506

u8 count = cpudata->floor_perf_cnt;

drivers/cpufreq/amd-pstate.c

1876

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

1890

cpudata = kzalloc_obj(*cpudata);

drivers/cpufreq/amd-pstate.c

1891

if (!cpudata)

drivers/cpufreq/amd-pstate.c

1894

cpudata->cpu = policy->cpu;

drivers/cpufreq/amd-pstate.c

1896

ret = amd_pstate_init_perf(cpudata);

drivers/cpufreq/amd-pstate.c

1900

amd_pstate_init_prefcore(cpudata);

drivers/cpufreq/amd-pstate.c

1902

ret = amd_pstate_init_freq(cpudata);

drivers/cpufreq/amd-pstate.c

1906

ret = amd_pstate_init_boost_support(cpudata);

drivers/cpufreq/amd-pstate.c

1910

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1912

policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq,

drivers/cpufreq/amd-pstate.c

1914

policy->cpuinfo.max_freq = cpudata->max_freq;

drivers/cpufreq/amd-pstate.c

1915

policy->driver_data = cpudata;

drivers/cpufreq/amd-pstate.c

1925

policy->boost_supported = READ_ONCE(cpudata->boost_supported);

drivers/cpufreq/amd-pstate.c

1934

cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata);

drivers/cpufreq/amd-pstate.c

1935

cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE;

drivers/cpufreq/amd-pstate.c

1938

cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE;

drivers/cpufreq/amd-pstate.c

1939

cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE;

drivers/cpufreq/amd-pstate.c

1940

cpudata->current_profile = PLATFORM_PROFILE_BALANCED;

drivers/cpufreq/amd-pstate.c

1946

ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc);

drivers/cpufreq/amd-pstate.c

1962

kfree(cpudata);

drivers/cpufreq/amd-pstate.c

1969

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1971

if (cpudata) {

drivers/cpufreq/amd-pstate.c

1972

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

1974

if (cpudata->dynamic_epp)

drivers/cpufreq/amd-pstate.c

1979

amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);

drivers/cpufreq/amd-pstate.c

1981

kfree(cpudata);

drivers/cpufreq/amd-pstate.c

1990

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

1995

policy->min != cpudata->min_limit_freq ||

drivers/cpufreq/amd-pstate.c

1996

policy->max != cpudata->max_limit_freq)

drivers/cpufreq/amd-pstate.c

1999

if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)

drivers/cpufreq/amd-pstate.c

2002

epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

2004

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2012

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2018

cpudata->policy = policy->policy;

drivers/cpufreq/amd-pstate.c

202

static u8 msr_get_epp(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

2035

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2036

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2044

cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);

drivers/cpufreq/amd-pstate.c

2050

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2051

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2060

FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2061

FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2062

FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2067

return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);

drivers/cpufreq/amd-pstate.c

207

ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);

drivers/cpufreq/amd-pstate.c

2072

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2073

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2082

FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2083

FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2084

FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

2089

ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);

drivers/cpufreq/amd-pstate.c

2094

cpudata->suspended = true;

drivers/cpufreq/amd-pstate.c

2101

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2102

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2103

int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur);

drivers/cpufreq/amd-pstate.c

2113

cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);

drivers/cpufreq/amd-pstate.c

2119

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

2120

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

2123

if (cpudata->suspended) {

drivers/cpufreq/amd-pstate.c

2131

cpudata->suspended = false;

drivers/cpufreq/amd-pstate.c

2134

cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq);

drivers/cpufreq/amd-pstate.c

218

static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

220

return static_call(amd_pstate_get_epp)(cpudata);

drivers/cpufreq/amd-pstate.c

223

static u8 shmem_get_epp(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

228

ret = cppc_get_epp_perf(cpudata->cpu, &epp);

drivers/cpufreq/amd-pstate.c

240

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

243

value = prev = READ_ONCE(cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

251

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

253

trace_call__amd_pstate_epp_perf(cpudata->cpu,

drivers/cpufreq/amd-pstate.c

268

int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);

drivers/cpufreq/amd-pstate.c

274

WRITE_ONCE(cpudata->cppc_req_cached, value);

drivers/cpufreq/amd-pstate.c

292

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

296

value = prev = READ_ONCE(cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

300

union perf_cached perf = cpudata->perf;

drivers/cpufreq/amd-pstate.c

302

trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,

drivers/cpufreq/amd-pstate.c

305

cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

307

cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

315

ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);

drivers/cpufreq/amd-pstate.c

322

WRITE_ONCE(cpudata->cppc_req_cached, value);

drivers/cpufreq/amd-pstate.c

336

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

344

value = prev = READ_ONCE(cpudata->cppc_req2_cached);

drivers/cpufreq/amd-pstate.c

353

ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value);

drivers/cpufreq/amd-pstate.c

360

WRITE_ONCE(cpudata->cppc_req2_cached, value);

drivers/cpufreq/amd-pstate.c

364

trace_call__amd_pstate_cppc_req2(cpudata->cpu, perf, changed,

drivers/cpufreq/amd-pstate.c

371

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

379

ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value);

drivers/cpufreq/amd-pstate.c

385

WRITE_ONCE(cpudata->cppc_req2_cached, value);

drivers/cpufreq/amd-pstate.c

387

cpudata->cppc_req2_cached);

drivers/cpufreq/amd-pstate.c

390

if (floor_perf < cpudata->perf.lowest_perf) {

drivers/cpufreq/amd-pstate.c

391

floor_perf = cpudata->perf.nominal_perf;

drivers/cpufreq/amd-pstate.c

398

cpudata->bios_floor_perf = floor_perf;

drivers/cpufreq/amd-pstate.c

399

cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq,

drivers/cpufreq/amd-pstate.c

406

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

413

epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

415

union perf_cached perf = cpudata->perf;

drivers/cpufreq/amd-pstate.c

417

trace_call__amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf,

drivers/cpufreq/amd-pstate.c

420

cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

422

cpudata->cppc_req_cached),

drivers/cpufreq/amd-pstate.c

431

ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);

drivers/cpufreq/amd-pstate.c

437

value = READ_ONCE(cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

439

WRITE_ONCE(cpudata->cppc_req_cached, value);

drivers/cpufreq/amd-pstate.c

461

static int msr_init_perf(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

463

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

467

int ret = rdmsrq_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,

drivers/cpufreq/amd-pstate.c

472

ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);

drivers/cpufreq/amd-pstate.c

476

ret = rdmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req);

drivers/cpufreq/amd-pstate.c

480

WRITE_ONCE(cpudata->cppc_req_cached, cppc_req);

drivers/cpufreq/amd-pstate.c

498

WRITE_ONCE(cpudata->perf, perf);

drivers/cpufreq/amd-pstate.c

499

WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1));

drivers/cpufreq/amd-pstate.c

500

WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1));

drivers/cpufreq/amd-pstate.c

505

static int shmem_init_perf(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

508

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

512

int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);

drivers/cpufreq/amd-pstate.c

516

ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);

drivers/cpufreq/amd-pstate.c

526

WRITE_ONCE(cpudata->perf, perf);

drivers/cpufreq/amd-pstate.c

527

WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf);

drivers/cpufreq/amd-pstate.c

532

ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel);

drivers/cpufreq/amd-pstate.c

538

ret = cppc_set_auto_sel(cpudata->cpu,

drivers/cpufreq/amd-pstate.c

549

static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

551

return static_call(amd_pstate_init_perf)(cpudata);

drivers/cpufreq/amd-pstate.c

557

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

569

value = prev = READ_ONCE(cpudata->cppc_req_cached);

drivers/cpufreq/amd-pstate.c

577

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

579

trace_call__amd_pstate_epp_perf(cpudata->cpu,

drivers/cpufreq/amd-pstate.c

595

ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);

drivers/cpufreq/amd-pstate.c

599

WRITE_ONCE(cpudata->cppc_req_cached, value);

drivers/cpufreq/amd-pstate.c

604

static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

614

if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {

drivers/cpufreq/amd-pstate.c

621

cpudata->cur.aperf = aperf;

drivers/cpufreq/amd-pstate.c

622

cpudata->cur.mperf = mperf;

drivers/cpufreq/amd-pstate.c

623

cpudata->cur.tsc = tsc;

drivers/cpufreq/amd-pstate.c

624

cpudata->cur.aperf -= cpudata->prev.aperf;

drivers/cpufreq/amd-pstate.c

625

cpudata->cur.mperf -= cpudata->prev.mperf;

drivers/cpufreq/amd-pstate.c

626

cpudata->cur.tsc -= cpudata->prev.tsc;

drivers/cpufreq/amd-pstate.c

628

cpudata->prev.aperf = aperf;

drivers/cpufreq/amd-pstate.c

629

cpudata->prev.mperf = mperf;

drivers/cpufreq/amd-pstate.c

630

cpudata->prev.tsc = tsc;

drivers/cpufreq/amd-pstate.c

632

cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);

drivers/cpufreq/amd-pstate.c

640

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

641

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

644

if (!cpudata->boost_supported)

drivers/cpufreq/amd-pstate.c

649

policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf);

drivers/cpufreq/amd-pstate.c

656

if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {

drivers/cpufreq/amd-pstate.c

657

trace_call__amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,

drivers/cpufreq/amd-pstate.c

658

cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,

drivers/cpufreq/amd-pstate.c

659

cpudata->cpu, fast_switch);

drivers/cpufreq/amd-pstate.c

675

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

681

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

682

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

685

policy_data->min = perf_to_freq(perf, cpudata->nominal_freq,

drivers/cpufreq/amd-pstate.c

688

policy_data->min = cpudata->lowest_nonlinear_freq;

drivers/cpufreq/amd-pstate.c

698

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

699

union perf_cached perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

701

perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max);

drivers/cpufreq/amd-pstate.c

702

WRITE_ONCE(cpudata->max_limit_freq, policy->max);

drivers/cpufreq/amd-pstate.c

704

if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {

drivers/cpufreq/amd-pstate.c

719

WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq));

drivers/cpufreq/amd-pstate.c

721

perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min);

drivers/cpufreq/amd-pstate.c

722

WRITE_ONCE(cpudata->min_limit_freq, policy->min);

drivers/cpufreq/amd-pstate.c

725

WRITE_ONCE(cpudata->perf, perf);

drivers/cpufreq/amd-pstate.c

732

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

736

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

738

if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)

drivers/cpufreq/amd-pstate.c

741

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

746

des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq);

drivers/cpufreq/amd-pstate.c

788

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

794

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

796

if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)

drivers/cpufreq/amd-pstate.c

799

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/amd-pstate.c

824

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

828

nominal_freq = READ_ONCE(cpudata->nominal_freq);

drivers/cpufreq/amd-pstate.c

831

policy->cpuinfo.max_freq = cpudata->max_freq;

drivers/cpufreq/amd-pstate.c

836

ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);

drivers/cpufreq/amd-pstate.c

838

pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu);

drivers/cpufreq/amd-pstate.c

846

struct amd_cpudata *cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

849

if (!cpudata->boost_supported) {

drivers/cpufreq/amd-pstate.c

860

static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

875

ret = rdmsrq_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val);

drivers/cpufreq/amd-pstate.c

883

cpudata->boost_supported = true;

drivers/cpufreq/amd-pstate.c

888

cpudata->boost_supported = false;

drivers/cpufreq/amd-pstate.c

899

static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

912

cpudata->hw_prefcore = true;

drivers/cpufreq/amd-pstate.c

915

sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu);

drivers/cpufreq/amd-pstate.c

920

struct amd_cpudata *cpudata;

drivers/cpufreq/amd-pstate.c

931

cpudata = policy->driver_data;

drivers/cpufreq/amd-pstate.c

933

prev_high = READ_ONCE(cpudata->prefcore_ranking);

drivers/cpufreq/amd-pstate.c

936

WRITE_ONCE(cpudata->prefcore_ranking, cur_high);

drivers/cpufreq/amd-pstate.c

987

static int amd_pstate_init_freq(struct amd_cpudata *cpudata)

drivers/cpufreq/amd-pstate.c

994

ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);

drivers/cpufreq/amd-pstate.c

997

perf = READ_ONCE(cpudata->perf);

drivers/cpufreq/intel_pstate.c

1004

if (!hybrid_max_perf_cpu || cpudata->pd_registered)

drivers/cpufreq/intel_pstate.c

1015

cpudata->pd_registered = true;

drivers/cpufreq/intel_pstate.c

1028

static void hybrid_update_perf_domain(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1036

static inline void hybrid_update_perf_domain(struct cpudata *cpu) {}

drivers/cpufreq/intel_pstate.c

1039

static void hybrid_set_cpu_capacity(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1059

static void hybrid_get_capacity_perf(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1076

struct cpudata *cpu = all_cpu_data[cpunum];

drivers/cpufreq/intel_pstate.c

1085

struct cpudata *max_perf_cpu = NULL;

drivers/cpufreq/intel_pstate.c

1090

struct cpudata *cpu = all_cpu_data[cpunum];

drivers/cpufreq/intel_pstate.c

1188

static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1198

static void intel_pstate_get_hwp_cap(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1216

static void hybrid_update_capacity(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1263

struct cpudata *cpu_data = all_cpu_data[cpu];

drivers/cpufreq/intel_pstate.c

1323

static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);

drivers/cpufreq/intel_pstate.c

1325

static void intel_pstate_hwp_offline(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1411

static void intel_pstate_hwp_enable(struct cpudata *cpudata);

drivers/cpufreq/intel_pstate.c

1413

static void intel_pstate_hwp_reenable(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

1421

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

1435

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

1468

struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

1473

intel_pstate_get_hwp_cap(cpudata);

drivers/cpufreq/intel_pstate.c

1476

cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;

drivers/cpufreq/intel_pstate.c

1494

struct cpudata *cpudata = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

1496

__intel_pstate_update_max_freq(policy, cpudata);

drivers/cpufreq/intel_pstate.c

1498

hybrid_update_capacity(cpudata);

drivers/cpufreq/intel_pstate.c

1553

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

1575

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

1633

struct cpudata *cpu = all_cpu_data[0];

drivers/cpufreq/intel_pstate.c

1651

struct cpudata *cpudata = all_cpu_data[cpu];

drivers/cpufreq/intel_pstate.c

1664

intel_pstate_get_hwp_cap(cpudata);

drivers/cpufreq/intel_pstate.c

1666

freq = cpudata->pstate.turbo_freq;

drivers/cpufreq/intel_pstate.c

1912

struct cpudata *cpudata =

drivers/cpufreq/intel_pstate.c

1913

container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);

drivers/cpufreq/intel_pstate.c

1915

if (intel_pstate_update_max_freq(cpudata->cpu)) {

drivers/cpufreq/intel_pstate.c

1921

hybrid_update_capacity(cpudata);

drivers/cpufreq/intel_pstate.c

1924

wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);

drivers/cpufreq/intel_pstate.c

1967

static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

1975

wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);

drivers/cpufreq/intel_pstate.c

1978

cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask);

drivers/cpufreq/intel_pstate.c

1982

cancel_delayed_work_sync(&cpudata->hwp_notify_work);

drivers/cpufreq/intel_pstate.c

1988

static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

1995

INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);

drivers/cpufreq/intel_pstate.c

1996

cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);

drivers/cpufreq/intel_pstate.c

2003

wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask);

drivers/cpufreq/intel_pstate.c

2004

wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);

drivers/cpufreq/intel_pstate.c

2008

static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

2010

cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);

drivers/cpufreq/intel_pstate.c

2018

if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE &&

drivers/cpufreq/intel_pstate.c

2019

cpudata->epp_default > HWP_EPP_PERFORMANCE) {

drivers/cpufreq/intel_pstate.c

2020

epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;

drivers/cpufreq/intel_pstate.c

2035

cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];

drivers/cpufreq/intel_pstate.c

2036

intel_pstate_set_epp(cpudata, cpudata->epp_default);

drivers/cpufreq/intel_pstate.c

2039

static void intel_pstate_hwp_enable(struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

2043

wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);

drivers/cpufreq/intel_pstate.c

2045

wrmsrq_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);

drivers/cpufreq/intel_pstate.c

2047

intel_pstate_enable_hwp_interrupt(cpudata);

drivers/cpufreq/intel_pstate.c

2049

if (cpudata->epp_default >= 0)

drivers/cpufreq/intel_pstate.c

2052

intel_pstate_update_epp_defaults(cpudata);

drivers/cpufreq/intel_pstate.c

2091

static u64 atom_get_val(struct cpudata *cpudata, int pstate)

drivers/cpufreq/intel_pstate.c

2097

vid_fp = cpudata->vid.min + mul_fp(

drivers/cpufreq/intel_pstate.c

2098

int_tofp(pstate - cpudata->pstate.min_pstate),

drivers/cpufreq/intel_pstate.c

2099

cpudata->vid.ratio);

drivers/cpufreq/intel_pstate.c

2101

vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);

drivers/cpufreq/intel_pstate.c

2104

if (pstate > cpudata->pstate.max_pstate)

drivers/cpufreq/intel_pstate.c

2105

vid = cpudata->vid.turbo;

drivers/cpufreq/intel_pstate.c

2141

static void atom_get_vid(struct cpudata *cpudata)

drivers/cpufreq/intel_pstate.c

2146

cpudata->vid.min = int_tofp((value >> 8) & 0x7f);

drivers/cpufreq/intel_pstate.c

2147

cpudata->vid.max = int_tofp((value >> 16) & 0x7f);

drivers/cpufreq/intel_pstate.c

2148

cpudata->vid.ratio = div_fp(

drivers/cpufreq/intel_pstate.c

2149

cpudata->vid.max - cpudata->vid.min,

drivers/cpufreq/intel_pstate.c

2150

int_tofp(cpudata->pstate.max_pstate -

drivers/cpufreq/intel_pstate.c

2151

cpudata->pstate.min_pstate));

drivers/cpufreq/intel_pstate.c

2154

cpudata->vid.turbo = value & 0x7f;

drivers/cpufreq/intel_pstate.c

2254

static u64 core_get_val(struct cpudata *cpudata, int pstate)

drivers/cpufreq/intel_pstate.c

2302

static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)

drivers/cpufreq/intel_pstate.c

2315

static void intel_pstate_set_min_pstate(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2320

static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2371

static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2418

static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2434

static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,

drivers/cpufreq/intel_pstate.c

2466

struct cpudata *cpu = container_of(data, struct cpudata, update_util);

drivers/cpufreq/intel_pstate.c

2474

static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2481

static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)

drivers/cpufreq/intel_pstate.c

2523

static inline int32_t get_avg_frequency(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2528

static inline int32_t get_avg_pstate(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2534

static inline int32_t get_target_pstate(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2569

static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)

drivers/cpufreq/intel_pstate.c

2577

static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)

drivers/cpufreq/intel_pstate.c

2586

static void intel_pstate_adjust_pstate(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2612

struct cpudata *cpu = container_of(data, struct cpudata, update_util);

drivers/cpufreq/intel_pstate.c

270

static struct cpudata **all_cpu_data;

drivers/cpufreq/intel_pstate.c

2747

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

2789

struct cpudata *cpu = all_cpu_data[cpu_num];

drivers/cpufreq/intel_pstate.c

2808

struct cpudata *cpu_data = all_cpu_data[cpu];

drivers/cpufreq/intel_pstate.c

2818

static int intel_pstate_get_max_freq(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

2824

static void intel_pstate_update_perf_limits(struct cpudata *cpu,

drivers/cpufreq/intel_pstate.c

2889

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

2938

static void intel_pstate_adjust_policy_max(struct cpudata *cpu,

drivers/cpufreq/intel_pstate.c

295

u64 (*get_val)(struct cpudata*, int pstate);

drivers/cpufreq/intel_pstate.c

2950

static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,

drivers/cpufreq/intel_pstate.c

296

void (*get_vid)(struct cpudata *);

drivers/cpufreq/intel_pstate.c

2976

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3003

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3039

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

3077

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3101

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3125

static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)

drivers/cpufreq/intel_pstate.c

3147

static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,

drivers/cpufreq/intel_pstate.c

3171

static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,

drivers/cpufreq/intel_pstate.c

3185

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3211

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3233

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3248

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3291

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

3380

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

3766

static struct cpudata **_all_cpu_data;

drivers/cpufreq/intel_pstate.c

444

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

500

struct cpudata *cpu;

drivers/cpufreq/intel_pstate.c

535

static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq,

drivers/cpufreq/intel_pstate.c

554

static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq)

drivers/cpufreq/intel_pstate.c

570

static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)

drivers/cpufreq/intel_pstate.c

618

struct cpudata *cpu = all_cpu_data[0];

drivers/cpufreq/intel_pstate.c

625

static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)

drivers/cpufreq/intel_pstate.c

682

static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)

drivers/cpufreq/intel_pstate.c

720

static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)

drivers/cpufreq/intel_pstate.c

746

static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,

drivers/cpufreq/intel_pstate.c

797

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

865

struct cpudata *cpu_data = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

882

struct cpudata *cpu = all_cpu_data[policy->cpu];

drivers/cpufreq/intel_pstate.c

924

static struct cpudata *hybrid_max_perf_cpu __read_mostly;

drivers/cpufreq/intel_pstate.c

996

struct cpudata *cpudata = all_cpu_data[cpu];

drivers/hwtracing/coresight/coresight-trbe.c

1022

*this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;

drivers/hwtracing/coresight/coresight-trbe.c

1034

struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);

drivers/hwtracing/coresight/coresight-trbe.c

1038

WARN_ON(cpudata->cpu != smp_processor_id());

drivers/hwtracing/coresight/coresight-trbe.c

1039

WARN_ON(cpudata->drvdata != drvdata);

drivers/hwtracing/coresight/coresight-trbe.c

1043

cpudata->buf = buf;

drivers/hwtracing/coresight/coresight-trbe.c

1044

cpudata->mode = mode;

drivers/hwtracing/coresight/coresight-trbe.c

1045

buf->cpudata = cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

1053

struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);

drivers/hwtracing/coresight/coresight-trbe.c

1054

struct trbe_buf *buf = cpudata->buf;

drivers/hwtracing/coresight/coresight-trbe.c

1056

WARN_ON(buf->cpudata != cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1057

WARN_ON(cpudata->cpu != smp_processor_id());

drivers/hwtracing/coresight/coresight-trbe.c

1058

WARN_ON(cpudata->drvdata != drvdata);

drivers/hwtracing/coresight/coresight-trbe.c

1059

if (cpudata->mode != CS_MODE_PERF)

drivers/hwtracing/coresight/coresight-trbe.c

1062

trbe_drain_and_disable_local(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1063

buf->cpudata = NULL;

drivers/hwtracing/coresight/coresight-trbe.c

1064

cpudata->buf = NULL;

drivers/hwtracing/coresight/coresight-trbe.c

1065

cpudata->mode = CS_MODE_DISABLED;

drivers/hwtracing/coresight/coresight-trbe.c

1079

set_trbe_enabled(buf->cpudata, trblimitr);

drivers/hwtracing/coresight/coresight-trbe.c

1103

trbe_drain_and_disable_local(buf->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1104

*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;

drivers/hwtracing/coresight/coresight-trbe.c

1114

struct trbe_cpudata *cpudata = buf->cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

1115

struct trbe_drvdata *drvdata = cpudata->drvdata;

drivers/hwtracing/coresight/coresight-trbe.c

1121

if (cpudata->mode != CS_MODE_PERF)

drivers/hwtracing/coresight/coresight-trbe.c

1124

if (cpudata->cpu != cpu)

drivers/hwtracing/coresight/coresight-trbe.c

1168

trbe_drain_and_disable_local(buf->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1209

struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);

drivers/hwtracing/coresight/coresight-trbe.c

1210

struct trbe_save_state *state = &cpudata->save_state;

drivers/hwtracing/coresight/coresight-trbe.c

1216

trbe_drain_and_disable_local(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1226

struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);

drivers/hwtracing/coresight/coresight-trbe.c

1227

struct trbe_save_state *state = &cpudata->save_state;

drivers/hwtracing/coresight/coresight-trbe.c

1243

set_trbe_enabled(cpudata, state->trblimitr);

drivers/hwtracing/coresight/coresight-trbe.c

1263

struct trbe_cpudata *cpudata = dev_get_drvdata(dev);

drivers/hwtracing/coresight/coresight-trbe.c

1265

return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);

drivers/hwtracing/coresight/coresight-trbe.c

1271

struct trbe_cpudata *cpudata = dev_get_drvdata(dev);

drivers/hwtracing/coresight/coresight-trbe.c

1273

return sprintf(buf, "%d\n", cpudata->trbe_flag);

drivers/hwtracing/coresight/coresight-trbe.c

1295

struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1297

trbe_reset_local(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1304

struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1307

trbe_reset_local(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1313

struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);

drivers/hwtracing/coresight/coresight-trbe.c

1322

if (WARN_ON(!cpudata->drvdata))

drivers/hwtracing/coresight/coresight-trbe.c

1325

dev = &cpudata->drvdata->pdev->dev;

drivers/hwtracing/coresight/coresight-trbe.c

1355

dev_set_drvdata(&trbe_csdev->dev, cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1369

struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);

drivers/hwtracing/coresight/coresight-trbe.c

1372

if (WARN_ON(!cpudata))

drivers/hwtracing/coresight/coresight-trbe.c

1386

cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);

drivers/hwtracing/coresight/coresight-trbe.c

1387

if (cpudata->trbe_hw_align > SZ_2K) {

drivers/hwtracing/coresight/coresight-trbe.c

1396

trbe_check_errata(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

1398

if (trbe_is_broken(cpudata)) {

drivers/hwtracing/coresight/coresight-trbe.c

1414

if (trbe_may_overwrite_in_fill_mode(cpudata))

drivers/hwtracing/coresight/coresight-trbe.c

1415

cpudata->trbe_align = PAGE_SIZE;

drivers/hwtracing/coresight/coresight-trbe.c

1417

cpudata->trbe_align = cpudata->trbe_hw_align;

drivers/hwtracing/coresight/coresight-trbe.c

1419

cpudata->trbe_flag = get_trbe_flag_update(trbidr);

drivers/hwtracing/coresight/coresight-trbe.c

1420

cpudata->cpu = cpu;

drivers/hwtracing/coresight/coresight-trbe.c

1421

cpudata->drvdata = drvdata;

drivers/hwtracing/coresight/coresight-trbe.c

1441

drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));

drivers/hwtracing/coresight/coresight-trbe.c

1442

if (!drvdata->cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

1465

free_percpu(drvdata->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

156

struct trbe_cpudata __percpu *cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

165

static void trbe_check_errata(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

175

set_bit(i, cpudata->errata);

drivers/hwtracing/coresight/coresight-trbe.c

179

static bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)

drivers/hwtracing/coresight/coresight-trbe.c

181

return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);

drivers/hwtracing/coresight/coresight-trbe.c

184

static bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

186

return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);

drivers/hwtracing/coresight/coresight-trbe.c

189

static bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

191

return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);

drivers/hwtracing/coresight/coresight-trbe.c

194

static bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

201

return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);

drivers/hwtracing/coresight/coresight-trbe.c

204

static bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

212

return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);

drivers/hwtracing/coresight/coresight-trbe.c

215

static bool trbe_is_broken(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

217

return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);

drivers/hwtracing/coresight/coresight-trbe.c

233

static void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)

drivers/hwtracing/coresight/coresight-trbe.c

246

if (trbe_needs_ctxt_sync_after_enable(cpudata))

drivers/hwtracing/coresight/coresight-trbe.c

250

static void set_trbe_disabled(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

262

if (trbe_needs_drain_after_disable(cpudata))

drivers/hwtracing/coresight/coresight-trbe.c

267

static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

270

set_trbe_disabled(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

273

static void trbe_reset_local(struct trbe_cpudata *cpudata)

drivers/hwtracing/coresight/coresight-trbe.c

313

trbe_drain_and_disable_local(buf->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

316

*this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;

drivers/hwtracing/coresight/coresight-trbe.c

393

struct trbe_cpudata *cpudata = buf->cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

403

if (trbe_may_write_out_of_range(cpudata))

drivers/hwtracing/coresight/coresight-trbe.c

421

struct trbe_cpudata *cpudata = buf->cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

444

if (!IS_ALIGNED(head, cpudata->trbe_align)) {

drivers/hwtracing/coresight/coresight-trbe.c

445

unsigned long delta = roundup(head, cpudata->trbe_align) - head;

drivers/hwtracing/coresight/coresight-trbe.c

647

set_trbe_enabled(buf->cpudata, trblimitr);

drivers/hwtracing/coresight/coresight-trbe.c

655

set_trbe_disabled(buf->cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

674

struct trbe_cpudata *cpudata = buf->cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

689

(trbe_may_overwrite_in_fill_mode(cpudata) ||

drivers/hwtracing/coresight/coresight-trbe.c

72

struct trbe_cpudata *cpudata;

drivers/hwtracing/coresight/coresight-trbe.c

741

if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&

drivers/hwtracing/coresight/coresight-trbe.c

806

struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);

drivers/hwtracing/coresight/coresight-trbe.c

813

WARN_ON(buf->cpudata != cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

814

WARN_ON(cpudata->cpu != smp_processor_id());

drivers/hwtracing/coresight/coresight-trbe.c

815

WARN_ON(cpudata->drvdata != drvdata);

drivers/hwtracing/coresight/coresight-trbe.c

816

if (cpudata->mode != CS_MODE_PERF)

drivers/hwtracing/coresight/coresight-trbe.c

850

trbe_drain_and_disable_local(cpudata);

drivers/hwtracing/coresight/coresight-trbe.c

964

if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {

drivers/hwtracing/coresight/coresight-trbe.c

988

if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {