lib/test_hmm.c

1014

xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);

lib/test_hmm.c

1039

static int dmirror_migrate_to_system(struct dmirror *dmirror,

lib/test_hmm.c

1044

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1081

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1082

args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;

lib/test_hmm.c

1089

dmirror_devmem_fault_alloc_and_copy(&args, dmirror);

lib/test_hmm.c

1104

static int dmirror_migrate_to_device(struct dmirror *dmirror,

lib/test_hmm.c

1109

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1155

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1163

dmirror_migrate_alloc_and_copy(&args, dmirror);

lib/test_hmm.c

1165

dmirror_migrate_finalize_and_map(&args, dmirror);

lib/test_hmm.c

1178

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1179

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

1180

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1199

static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,

lib/test_hmm.c

1216

if (dmirror->mdevice == dmirror_page_to_device(page))

lib/test_hmm.c

1222

if (dmirror->mdevice == dmirror_page_to_device(page))

lib/test_hmm.c

1246

struct dmirror *dmirror = dmi->dmirror;

lib/test_hmm.c

1249

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1250

else if (!mutex_trylock(&dmirror->mutex))

lib/test_hmm.c

1259

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1267

static int dmirror_range_snapshot(struct dmirror *dmirror,

lib/test_hmm.c

1271

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1279

notifier.dmirror = dmirror;

lib/test_hmm.c

1305

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1308

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1316

dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);

lib/test_hmm.c

1318

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1324

static int dmirror_snapshot(struct dmirror *dmirror,

lib/test_hmm.c

1327

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1337

.dev_private_owner = dmirror->mdevice,

lib/test_hmm.c

1362

ret = dmirror_range_snapshot(dmirror, &range, perm);

lib/test_hmm.c

1484

struct dmirror *dmirror;

lib/test_hmm.c

1487

dmirror = filp->private_data;

lib/test_hmm.c

1488

if (!dmirror)

lib/test_hmm.c

149

dmirror_select_device(struct dmirror *dmirror)

lib/test_hmm.c

1504

ret = dmirror_read(dmirror, &cmd);

lib/test_hmm.c

1508

ret = dmirror_write(dmirror, &cmd);

lib/test_hmm.c

151

return (dmirror->mdevice->zone_device_type ==

lib/test_hmm.c

1512

ret = dmirror_migrate_to_device(dmirror, &cmd);

lib/test_hmm.c

1516

ret = dmirror_migrate_to_system(dmirror, &cmd);

lib/test_hmm.c

1520

ret = dmirror_exclusive(dmirror, &cmd);

lib/test_hmm.c

1524

ret = dmirror_check_atomic(dmirror, cmd.addr,

lib/test_hmm.c

1529

ret = dmirror_snapshot(dmirror, &cmd);

lib/test_hmm.c

1533

dmirror_device_remove_chunks(dmirror->mdevice);

lib/test_hmm.c

1537

dmirror->flags = cmd.npages;

lib/test_hmm.c

1622

struct dmirror *dmirror;

lib/test_hmm.c

1632

dmirror = rpage->zone_device_data;

lib/test_hmm.c

165

struct dmirror *dmirror;

lib/test_hmm.c

1658

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1659

args.flags = dmirror_select_device(dmirror);

lib/test_hmm.c

1673

ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);

lib/test_hmm.c

169

dmirror = kzalloc_obj(*dmirror);

lib/test_hmm.c

170

if (dmirror == NULL)

lib/test_hmm.c

173

dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);

lib/test_hmm.c

174

mutex_init(&dmirror->mutex);

lib/test_hmm.c

175

xa_init(&dmirror->pt);

lib/test_hmm.c

177

ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,

lib/test_hmm.c

180

kfree(dmirror);

lib/test_hmm.c

184

filp->private_data = dmirror;

lib/test_hmm.c

190

struct dmirror *dmirror = filp->private_data;

lib/test_hmm.c

192

mmu_interval_notifier_remove(&dmirror->notifier);

lib/test_hmm.c

193

xa_destroy(&dmirror->pt);

lib/test_hmm.c

194

kfree(dmirror);

lib/test_hmm.c

210

static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)

lib/test_hmm.c

236

entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);

lib/test_hmm.c

244

static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

255

xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,

lib/test_hmm.c

257

xa_erase(&dmirror->pt, pfn);

lib/test_hmm.c

264

struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);

lib/test_hmm.c

271

range->owner == dmirror->mdevice)

lib/test_hmm.c

275

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

276

else if (!mutex_trylock(&dmirror->mutex))

lib/test_hmm.c

280

dmirror_do_update(dmirror, range->start, range->end);

lib/test_hmm.c

282

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

290

static int dmirror_range_fault(struct dmirror *dmirror,

lib/test_hmm.c

293

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

314

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

317

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

323

ret = dmirror_do_fault(dmirror, range);

lib/test_hmm.c

325

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

330

static int dmirror_fault(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

333

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

337

.notifier = &dmirror->notifier,

lib/test_hmm.c

342

.dev_private_owner = dmirror->mdevice,

lib/test_hmm.c

354

ret = dmirror_range_fault(dmirror, &range);

lib/test_hmm.c

363

static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

375

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

389

static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)

lib/test_hmm.c

406

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

407

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

408

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

413

ret = dmirror_fault(dmirror, start, end, false);

lib/test_hmm.c

429

static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

441

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

455

static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)

lib/test_hmm.c

477

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

478

ret = dmirror_do_write(dmirror, start, end, &bounce);

lib/test_hmm.c

479

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

484

ret = dmirror_fault(dmirror, start, end, true);

lib/test_hmm.c

628

static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror,

lib/test_hmm.c

634

struct dmirror_device *mdevice = dmirror->mdevice;

lib/test_hmm.c

678

struct dmirror *dmirror)

lib/test_hmm.c

705

if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {

lib/test_hmm.c

706

dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;

lib/test_hmm.c

709

dpage = dmirror_devmem_alloc_page(dmirror, is_large);

lib/test_hmm.c

728

dpage = dmirror_devmem_alloc_page(dmirror, false);

lib/test_hmm.c

730

rpage->zone_device_data = dmirror;

lib/test_hmm.c

754

rpage->zone_device_data = dmirror;

lib/test_hmm.c

797

static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

805

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

814

struct dmirror *dmirror)

lib/test_hmm.c

819

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

822

entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC);

lib/test_hmm.c

824

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

828

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

83

struct dmirror *dmirror;

lib/test_hmm.c

833

struct dmirror *dmirror)

lib/test_hmm.c

844

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

873

entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC);

lib/test_hmm.c

875

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

881

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

885

static int dmirror_exclusive(struct dmirror *dmirror,

lib/test_hmm.c

890

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

914

ret = dmirror_atomic_map(addr, page, dmirror);

lib/test_hmm.c

928

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

929

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

930

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

943

struct dmirror *dmirror)

lib/test_hmm.c

975

if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {

lib/test_hmm.c

976

dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;