lib/test_hmm.c

1005

struct dmirror *dmirror)

lib/test_hmm.c

1037

if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {

lib/test_hmm.c

1038

dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;

lib/test_hmm.c

1095

xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);

lib/test_hmm.c

1120

static int dmirror_migrate_to_system(struct dmirror *dmirror,

lib/test_hmm.c

1125

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1162

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1163

args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;

lib/test_hmm.c

1170

if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) {

lib/test_hmm.c

1189

static int dmirror_migrate_to_device(struct dmirror *dmirror,

lib/test_hmm.c

1194

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1240

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1248

dmirror_migrate_alloc_and_copy(&args, dmirror);

lib/test_hmm.c

1250

dmirror_migrate_finalize_and_map(&args, dmirror);

lib/test_hmm.c

1263

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1264

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

1265

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1284

static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,

lib/test_hmm.c

1301

if (dmirror->mdevice == dmirror_page_to_device(page))

lib/test_hmm.c

1307

if (dmirror->mdevice == dmirror_page_to_device(page))

lib/test_hmm.c

1331

struct dmirror *dmirror = dmi->dmirror;

lib/test_hmm.c

1334

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1335

else if (!mutex_trylock(&dmirror->mutex))

lib/test_hmm.c

1344

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1352

static int dmirror_range_snapshot(struct dmirror *dmirror,

lib/test_hmm.c

1356

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1364

notifier.dmirror = dmirror;

lib/test_hmm.c

1390

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

1393

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1401

dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);

lib/test_hmm.c

1403

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

1409

static int dmirror_snapshot(struct dmirror *dmirror,

lib/test_hmm.c

1412

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

1422

.dev_private_owner = dmirror->mdevice,

lib/test_hmm.c

1447

ret = dmirror_range_snapshot(dmirror, &range, perm);

lib/test_hmm.c

149

dmirror_select_device(struct dmirror *dmirror)

lib/test_hmm.c

151

return (dmirror->mdevice->zone_device_type ==

lib/test_hmm.c

1519

struct dmirror *dmirror;

lib/test_hmm.c

1522

dmirror = filp->private_data;

lib/test_hmm.c

1523

if (!dmirror)

lib/test_hmm.c

1539

ret = dmirror_read(dmirror, &cmd);

lib/test_hmm.c

1543

ret = dmirror_write(dmirror, &cmd);

lib/test_hmm.c

1547

ret = dmirror_migrate_to_device(dmirror, &cmd);

lib/test_hmm.c

1551

ret = dmirror_migrate_to_system(dmirror, &cmd);

lib/test_hmm.c

1555

ret = dmirror_exclusive(dmirror, &cmd);

lib/test_hmm.c

1559

ret = dmirror_check_atomic(dmirror, cmd.addr,

lib/test_hmm.c

1564

ret = dmirror_snapshot(dmirror, &cmd);

lib/test_hmm.c

1568

dmirror_device_remove_chunks(dmirror->mdevice);

lib/test_hmm.c

1572

dmirror->flags = cmd.npages;

lib/test_hmm.c

165

struct dmirror *dmirror;

lib/test_hmm.c

1657

struct dmirror *dmirror;

lib/test_hmm.c

1667

dmirror = rpage->zone_device_data;

lib/test_hmm.c

169

dmirror = kzalloc_obj(*dmirror);

lib/test_hmm.c

1693

args.pgmap_owner = dmirror->mdevice;

lib/test_hmm.c

1694

args.flags = dmirror_select_device(dmirror);

lib/test_hmm.c

170

if (dmirror == NULL)

lib/test_hmm.c

1714

ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);

lib/test_hmm.c

173

dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);

lib/test_hmm.c

174

mutex_init(&dmirror->mutex);

lib/test_hmm.c

175

xa_init(&dmirror->pt);

lib/test_hmm.c

177

ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,

lib/test_hmm.c

180

kfree(dmirror);

lib/test_hmm.c

184

filp->private_data = dmirror;

lib/test_hmm.c

240

struct dmirror *dmirror = filp->private_data;

lib/test_hmm.c

241

struct dmirror_device *mdevice = dmirror->mdevice;

lib/test_hmm.c

244

mmu_interval_notifier_remove(&dmirror->notifier);

lib/test_hmm.c

255

xa_destroy(&dmirror->pt);

lib/test_hmm.c

256

kfree(dmirror);

lib/test_hmm.c

272

static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)

lib/test_hmm.c

298

entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);

lib/test_hmm.c

306

static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

317

xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,

lib/test_hmm.c

319

xa_erase(&dmirror->pt, pfn);

lib/test_hmm.c

326

struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);

lib/test_hmm.c

333

range->owner == dmirror->mdevice)

lib/test_hmm.c

337

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

338

else if (!mutex_trylock(&dmirror->mutex))

lib/test_hmm.c

342

dmirror_do_update(dmirror, range->start, range->end);

lib/test_hmm.c

344

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

352

static int dmirror_range_fault(struct dmirror *dmirror,

lib/test_hmm.c

355

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

376

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

379

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

385

ret = dmirror_do_fault(dmirror, range);

lib/test_hmm.c

387

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

392

static int dmirror_fault(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

395

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

399

.notifier = &dmirror->notifier,

lib/test_hmm.c

404

.dev_private_owner = dmirror->mdevice,

lib/test_hmm.c

416

ret = dmirror_range_fault(dmirror, &range);

lib/test_hmm.c

425

static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

437

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

451

static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)

lib/test_hmm.c

468

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

469

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

470

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

475

ret = dmirror_fault(dmirror, start, end, false);

lib/test_hmm.c

491

static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

503

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

517

static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)

lib/test_hmm.c

539

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

540

ret = dmirror_do_write(dmirror, start, end, &bounce);

lib/test_hmm.c

541

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

546

ret = dmirror_fault(dmirror, start, end, true);

lib/test_hmm.c

690

static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror,

lib/test_hmm.c

696

struct dmirror_device *mdevice = dmirror->mdevice;

lib/test_hmm.c

740

struct dmirror *dmirror)

lib/test_hmm.c

767

if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {

lib/test_hmm.c

768

dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;

lib/test_hmm.c

771

dpage = dmirror_devmem_alloc_page(dmirror, is_large);

lib/test_hmm.c

790

dpage = dmirror_devmem_alloc_page(dmirror, false);

lib/test_hmm.c

792

rpage->zone_device_data = dmirror;

lib/test_hmm.c

816

rpage->zone_device_data = dmirror;

lib/test_hmm.c

83

struct dmirror *dmirror;

lib/test_hmm.c

859

static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,

lib/test_hmm.c

867

entry = xa_load(&dmirror->pt, pfn);

lib/test_hmm.c

876

struct dmirror *dmirror)

lib/test_hmm.c

881

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

884

entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC);

lib/test_hmm.c

886

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

890

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

895

struct dmirror *dmirror)

lib/test_hmm.c

906

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

935

entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC);

lib/test_hmm.c

937

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

943

mutex_unlock(&dmirror->mutex);

lib/test_hmm.c

947

static int dmirror_exclusive(struct dmirror *dmirror,

lib/test_hmm.c

952

struct mm_struct *mm = dmirror->notifier.mm;

lib/test_hmm.c

976

ret = dmirror_atomic_map(addr, page, dmirror);

lib/test_hmm.c

990

mutex_lock(&dmirror->mutex);

lib/test_hmm.c

991

ret = dmirror_do_read(dmirror, start, end, &bounce);

lib/test_hmm.c

992

mutex_unlock(&dmirror->mutex);