sys/arch/macppc/pci/mpcpcibus.c

142

struct ranges_32 *prange = (void *)range_store;

sys/arch/macppc/pci/mpcpcibus.c

156

if (prange[i].cspace == 0x01000000) {

sys/arch/macppc/pci/mpcpcibus.c

161

extent_free(sc->sc_ioex, prange[i].child_lo,

sys/arch/macppc/pci/mpcpcibus.c

162

prange[i].size_lo, EX_NOWAIT);

sys/arch/macppc/pci/mpcpcibus.c

166

if (prange[found].cspace == 0x01000000) {

sys/arch/macppc/pci/mpcpcibus.c

167

sc->sc_iobus_space.bus_base = prange[found].phys;

sys/arch/macppc/pci/mpcpcibus.c

168

sc->sc_iobus_space.bus_size = prange[found].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

181

if (prange[i].cspace == 0x02000000) {

sys/arch/macppc/pci/mpcpcibus.c

184

prange[i].phys,

sys/arch/macppc/pci/mpcpcibus.c

185

prange[i].size_lo);

sys/arch/macppc/pci/mpcpcibus.c

188

if ((base + size) == prange[i].phys)

sys/arch/macppc/pci/mpcpcibus.c

189

size += prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

191

base = prange[i].phys;

sys/arch/macppc/pci/mpcpcibus.c

192

size = prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

195

base = prange[i].phys;

sys/arch/macppc/pci/mpcpcibus.c

196

size = prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

200

extent_free(sc->sc_memex, prange[i].child_lo,

sys/arch/macppc/pci/mpcpcibus.c

201

prange[i].size_lo, EX_NOWAIT);

sys/arch/macppc/pci/mpcpcibus.c

225

struct ranges_64 *prange = (void *)range_store;

sys/arch/macppc/pci/mpcpcibus.c

236

if (prange[0].cspace == 0xabb10113) { /* appl U3; */

sys/arch/macppc/pci/mpcpcibus.c

237

prange[0].cspace = 0x01000000;

sys/arch/macppc/pci/mpcpcibus.c

238

prange[0].child_lo = 0x00000000;

sys/arch/macppc/pci/mpcpcibus.c

239

prange[0].phys_lo = 0xf8070000;

sys/arch/macppc/pci/mpcpcibus.c

240

prange[0].size_lo = 0x00001000;

sys/arch/macppc/pci/mpcpcibus.c

241

prange[1].cspace = 0x02000000;

sys/arch/macppc/pci/mpcpcibus.c

242

prange[1].child_lo = 0xf2000000;

sys/arch/macppc/pci/mpcpcibus.c

243

prange[1].phys_lo = 0xf2000000;

sys/arch/macppc/pci/mpcpcibus.c

244

prange[1].size_lo = 0x02800000;

sys/arch/macppc/pci/mpcpcibus.c

251

if (prange[i].cspace == 0x01000000) {

sys/arch/macppc/pci/mpcpcibus.c

256

extent_free(sc->sc_ioex, prange[i].child_lo,

sys/arch/macppc/pci/mpcpcibus.c

257

prange[i].size_lo, EX_NOWAIT);

sys/arch/macppc/pci/mpcpcibus.c

261

if (prange[found].cspace == 0x01000000) {

sys/arch/macppc/pci/mpcpcibus.c

262

sc->sc_iobus_space.bus_base = prange[found].phys_lo;

sys/arch/macppc/pci/mpcpcibus.c

263

sc->sc_iobus_space.bus_size = prange[found].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

276

if (prange[i].cspace == 0x02000000) {

sys/arch/macppc/pci/mpcpcibus.c

279

prange[i].phys_lo,

sys/arch/macppc/pci/mpcpcibus.c

280

prange[i].size_lo);

sys/arch/macppc/pci/mpcpcibus.c

283

if ((base + size) == prange[i].phys_lo) {

sys/arch/macppc/pci/mpcpcibus.c

284

size += prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

286

base = prange[i].phys_lo;

sys/arch/macppc/pci/mpcpcibus.c

287

size = prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

290

base = prange[i].phys_lo;

sys/arch/macppc/pci/mpcpcibus.c

291

size = prange[i].size_lo;

sys/arch/macppc/pci/mpcpcibus.c

295

extent_free(sc->sc_memex, prange[i].child_lo,

sys/arch/macppc/pci/mpcpcibus.c

296

prange[i].size_lo, EX_NOWAIT);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

1000

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

214

svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

219

svm_range_bo_ref(prange->svm_bo);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

220

page->zone_device_data = prange->svm_bo;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

278

svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

292

pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

293

prange->last, ttm_res_offset);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

298

amdgpu_res_first(prange->ttm_res, ttm_res_offset,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

306

svm_migrate_get_vram_page(prange, migrate->dst[i]);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

388

svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

392

struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

423

0, node->id, prange->prefetch_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

424

prange->preferred_loc, trigger);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

429

__func__, r, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

436

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

445

r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

464

pdd = svm_range_get_pdd_by_node(prange, node);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

488

svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

499

if (start_mgr < prange->start || last_mgr > prange->last) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

501

start_mgr, last_mgr, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

505

node = svm_range_get_node_by_id(prange, best_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

512

prange->svms, start_mgr, last_mgr, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

519

prange->npages * PAGE_SIZE,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

527

r = svm_range_vram_node_new(node, prange, true);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

532

ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

542

r = svm_migrate_vma_to_vram(node, prange, vma, addr, next, trigger, ttm_res_offset);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

554

prange->actual_loc = best_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

555

prange->vram_pages += mpages;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

556

} else if (!prange->actual_loc) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

560

svm_range_vram_node_free(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

565

prange->npages * PAGE_SIZE,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

582

svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

594

pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

595

prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

608

prange->svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

634

prange->svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

686

svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

690

struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

725

node->id, 0, prange->prefetch_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

726

prange->preferred_loc, trigger);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

731

__func__, r, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

738

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

747

r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

767

pdd = svm_range_get_pdd_by_node(prange, node);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

789

int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

802

if (!prange->actual_loc) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

804

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

808

if (start_mgr < prange->start || last_mgr > prange->last) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

810

start_mgr, last_mgr, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

814

node = svm_range_get_node_by_id(prange, prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

816

pr_debug("failed to get kfd node by id 0x%x\n", prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

820

prange->svms, prange, start_mgr, last_mgr,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

821

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

831

pr_debug("failed to find vma for prange %p\n", prange);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

837

r = svm_migrate_vma_to_ram(node, prange, vma, addr, next, trigger,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

840

pr_debug("failed %ld to migrate prange %p\n", r, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

849

WARN_ONCE(prange->vram_pages < mpages,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

851

prange->vram_pages, mpages);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

852

prange->vram_pages -= mpages;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

857

if (prange->vram_pages == 0 && prange->ttm_res) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

858

prange->actual_loc = 0;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

859

svm_range_vram_node_free(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

884

svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

895

pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

898

r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

902

} while (prange->actual_loc && --retries);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

904

if (prange->actual_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

907

return svm_migrate_ram_to_vram(prange, best_loc, start, last, mm, trigger);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

911

svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

915

if (!prange->actual_loc || prange->actual_loc == best_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

916

return svm_migrate_ram_to_vram(prange, best_loc, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

920

return svm_migrate_vram_to_vram(prange, best_loc, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

940

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

976

prange = svm_range_from_addr(&p->svms, addr, NULL);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

977

if (!prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

983

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

985

if (!prange->actual_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

989

size = 1UL << prange->granularity;

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

990

start = max(ALIGN_DOWN(addr, size), prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

991

last = min(ALIGN(addr + 1, size) - 1, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

993

r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.c

997

r, prange->svms, prange, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.h

43

int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_migrate.h

47

int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

110

prange = svm_range_from_addr(&p->svms, addr, NULL);

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

111

if (!prange)

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

114

if (!prange->mapped_to_gpu)

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

120

if (!test_bit(gpuidx, prange->bitmap_access) &&

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

121

!test_bit(gpuidx, prange->bitmap_aip))

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

124

if (!(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED))

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

127

list_add(&prange->update_list, &update_list);

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

129

if (prange->last - prange->start + 1 >= size) {

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

134

size -= prange->last - prange->start + 1;

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

135

addr += prange->last - prange->start + 1;

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

142

list_for_each_entry(prange, &update_list, update_list)

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

143

atomic_inc(&prange->queue_refcount);

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

154

struct svm_range *prange, *pchild;

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

168

prange = container_of(node, struct svm_range, it_node);

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

172

if (atomic_add_unless(&prange->queue_refcount, -1, 0)) {

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

173

list_for_each_entry(pchild, &prange->child_list, child_list)

sys/dev/pci/drm/amd/amdkfd/kfd_queue.c

94

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

100

list_del(&prange->svm_bo_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

101

spin_unlock(&prange->svm_bo->list_lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

104

list_del(&prange->list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

105

if (prange->it_node.start != 0 && prange->it_node.last != 0)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

106

interval_tree_remove(&prange->it_node, &prange->svms->objects);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

110

svm_range_add_notifier_locked(struct mm_struct *mm, struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1109

svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1112

uint64_t old_start = prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1113

uint64_t old_last = prange->last;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1117

pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

112

pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1125

svms = prange->svms;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

113

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1133

r = svm_range_split_adjust(*new, prange, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1145

svm_range_split_tail(struct svm_range *prange, uint64_t new_last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1148

unsigned long last_align_down = ALIGN_DOWN(prange->last, 512);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1149

unsigned long start_align = ALIGN(prange->start, 512);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

115

mmu_interval_notifier_insert_locked(&prange->notifier, mm,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1154

r = svm_range_split(prange, prange->start, new_last, &tail);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

116

prange->start << PAGE_SHIFT,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1169

svm_range_split_head(struct svm_range *prange, uint64_t new_start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

117

prange->npages << PAGE_SHIFT,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1172

unsigned long last_align_down = ALIGN_DOWN(prange->last, 512);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1173

unsigned long start_align = ALIGN(prange->start, 512);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1178

r = svm_range_split(prange, new_start, prange->last, &head);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1193

svm_range_add_child(struct svm_range *prange, struct svm_range *pchild, enum svm_work_list_ops op)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1196

pchild, pchild->start, pchild->last, prange, op);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1200

list_add_tail(&pchild->child_list, &prange->child_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1212

struct svm_range *prange, int domain)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1215

uint32_t flags = prange->flags;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1225

bo_node = prange->svm_bo->node;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

129

static void svm_range_add_to_svms(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

131

pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

132

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

134

list_move_tail(&prange->list, &prange->svms->list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1345

svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

135

prange->it_node.start = prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1355

if (!prange->mapped_to_gpu) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1357

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

136

prange->it_node.last = prange->last;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1361

if (prange->start == start && prange->last == last) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1362

pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1363

prange->mapped_to_gpu = false;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1366

bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1368

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

137

interval_tree_insert(&prange->it_node, &prange->svms->objects);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

140

static void svm_range_remove_notifier(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1401

svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1414

last_start = prange->start + offset;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1416

pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

143

prange->svms, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1431

last_start, prange->start + i, last_domain ? "GPU" : "CPU");

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1433

pte_flags = svm_range_get_pte_flags(pdd->dev, vm, prange, last_domain);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1438

prange->svms, last_start, prange->start + i,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

144

prange->notifier.interval_tree.start >> PAGE_SHIFT,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1447

NULL, last_start, prange->start + i,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1449

(last_start - prange->start) << PAGE_SHIFT,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

145

prange->notifier.interval_tree.last >> PAGE_SHIFT);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1453

for (j = last_start - prange->start; j <= i; j++)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1457

pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1460

last_start = prange->start + i + 1;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1466

prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

147

if (prange->notifier.interval_tree.start != 0 &&

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1478

svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

148

prange->notifier.interval_tree.last != 0)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1489

if (prange->svm_bo && prange->ttm_res)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

149

mmu_interval_notifier_remove(&prange->notifier);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1490

bo_adev = prange->svm_bo->node->adev;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1492

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1511

r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1512

prange->dma_addr[gpuidx],

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1536

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

160

svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1635

struct svm_range *prange, int32_t gpuidx,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1648

ctx->process = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1649

ctx->prange = prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

165

dma_addr_t *addr = prange->dma_addr[gpuidx];

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1656

bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1662

if (prange->actual_loc) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1664

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1667

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1671

if (test_bit(gpuidx, prange->bitmap_access))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1680

if (prange->mapped_to_gpu ||

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1681

prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1682

bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1685

bitmap_or(ctx->bitmap, prange->bitmap_access,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1686

prange->bitmap_aip, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1694

if (prange->actual_loc && !prange->ttm_res) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1707

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

171

addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

174

prange->dma_addr[gpuidx] = addr;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1742

svm_range_lock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1745

s = max(start, prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1746

e = min(end, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1748

r = svm_range_unmap_from_gpus(prange, s, e,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1750

svm_range_unlock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1759

r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1770

offset = (addr >> PAGE_SHIFT) - prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1771

r = svm_range_dma_map(prange, ctx->bitmap, offset, npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1777

svm_range_lock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1788

if (!r && !list_empty(&prange->child_list)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1794

map_start_vma = max(map_start, prange->start + offset);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1795

map_last_vma = min(map_last, prange->start + offset + npages - 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1797

offset = map_start_vma - prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1799

r = svm_range_map_to_gpus(prange, offset, npages, readonly,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1805

prange->mapped_to_gpu = true;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1807

svm_range_unlock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1814

prange->validate_timestamp = ktime_get_boottime();

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

184

struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1851

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1881

list_for_each_entry(prange, &svms->list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1882

invalid = atomic_read(&prange->invalid);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1887

prange->svms, prange, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1893

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1895

r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1899

prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1901

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1905

if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1958

svm_range_evict(struct svm_range *prange, struct mm_struct *mm,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1962

struct svm_range_list *svms = prange->svms;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1970

svms, prange->start, prange->last, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1973

(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1975

bool mapped = prange->mapped_to_gpu;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1977

list_for_each_entry(pchild, &prange->child_list, child_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1993

if (prange->start <= last && prange->last >= start)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

1994

atomic_inc(&prange->invalid);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2001

prange->svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2021

prange->svms, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2022

list_for_each_entry(pchild, &prange->child_list, child_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2030

s = max(start, prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2031

l = min(last, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2033

svm_range_unmap_from_gpus(prange, s, l, trigger);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

207

svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2101

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2110

prange = svm_range_new(svms, start, l, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2111

if (!prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2113

list_add(&prange->list, insert_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2114

list_add(&prange->update_list, update_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

215

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2160

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2181

prange = container_of(node, struct svm_range, it_node);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2185

if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2186

prange->mapped_to_gpu) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2193

struct svm_range *old = prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2195

prange = svm_range_clone(old);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2196

if (!prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2202

list_add(&prange->list, insert_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2203

list_add(&prange->update_list, update_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2207

r = svm_range_split_head(prange, start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2214

r = svm_range_split_tail(prange, last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2223

list_add(&prange->update_list, update_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2247

list_for_each_entry_safe(prange, tmp, insert_list, list)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2248

svm_range_free(prange, false);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2249

list_for_each_entry_safe(prange, tmp, &new_list, list)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2250

svm_range_free(prange, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2260

struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2265

start = prange->notifier.interval_tree.start >> PAGE_SHIFT;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2266

last = prange->notifier.interval_tree.last >> PAGE_SHIFT;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2268

if (prange->start == start && prange->last == last)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

227

r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2272

prange->svms, prange, start, last, prange->start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2273

prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2276

interval_tree_remove(&prange->it_node, &prange->svms->objects);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2277

svm_range_remove_notifier(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2279

prange->it_node.start = prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2280

prange->it_node.last = prange->last;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2282

interval_tree_insert(&prange->it_node, &prange->svms->objects);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2283

svm_range_add_notifier_locked(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2287

svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2290

switch (prange->work_item.op) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2293

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2297

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2298

svm_range_unlink(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2299

svm_range_remove_notifier(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2300

svm_range_free(prange, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2304

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2305

svm_range_update_notifier_and_interval_tree(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2309

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2310

svm_range_update_notifier_and_interval_tree(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2314

pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2315

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2316

svm_range_add_to_svms(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2317

svm_range_add_notifier_locked(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2321

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2322

svm_range_add_to_svms(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2323

svm_range_add_notifier_locked(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2327

WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2328

prange->work_item.op);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2368

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2376

prange = list_first_entry(&svms->deferred_range_list,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2380

pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2381

prange->start, prange->last, prange->work_item.op);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2383

mm = prange->work_item.mm;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2396

list_del_init(&prange->deferred_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2400

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2401

while (!list_empty(&prange->child_list)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2404

pchild = list_first_entry(&prange->child_list,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2411

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2413

svm_range_handle_list_op(svms, prange, mm);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2429

svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2434

if (!list_empty(&prange->deferred_list)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2435

pr_debug("update exist prange 0x%p work op %d\n", prange, op);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2436

WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2438

prange->work_item.op != SVM_OP_UNMAP_RANGE)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2439

prange->work_item.op = op;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2445

prange->work_item.mm = mm;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2446

prange->work_item.op = op;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2447

list_add_tail(&prange->deferred_list,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2448

&prange->svms->deferred_range_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2450

prange, prange->start, prange->last, op);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2465

svm_range_unmap_split(struct svm_range *parent, struct svm_range *prange, unsigned long start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2471

if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2472

pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2473

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2476

if (start > prange->last || last < prange->start)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2479

head = tail = prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2480

if (start > prange->start)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2481

svm_range_split(prange, prange->start, start - 1, &tail);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2485

if (head != prange && tail != prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2488

} else if (tail != prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2490

} else if (head != prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2492

} else if (parent != prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2493

prange->work_item.op = SVM_OP_UNMAP_RANGE;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2498

svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2509

if (atomic_read(&prange->queue_refcount)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2513

prange->start << PAGE_SHIFT);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2525

prange, prange->start, prange->last, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

254

void svm_range_dma_unmap(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2560

unmap_parent = start <= prange->start && last >= prange->last;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2562

list_for_each_entry(pchild, &prange->child_list, child_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2568

svm_range_unmap_split(prange, pchild, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2571

s = max(start, prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2572

l = min(last, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2574

svm_range_unmap_from_gpus(prange, s, l, trigger);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2575

svm_range_unmap_split(prange, prange, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2578

svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2580

svm_range_add_list_work(svms, prange, mm,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2612

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

262

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2629

prange = container_of(mni, struct svm_range, notifier);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2631

svm_range_lock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2636

svm_range_unmap_from_cpu(mni->mm, prange, start, last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2639

svm_range_evict(prange, mni->mm, start, last, range->event);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2643

svm_range_unlock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

265

dma_addr = prange->dma_addr[gpuidx];

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2663

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2670

prange = container_of(node, struct svm_range, it_node);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2672

addr, prange->start, prange->last, node->start, node->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2674

if (addr >= prange->start && addr <= prange->last) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2676

*parent = prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2677

return prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2679

list_for_each_entry(pchild, &prange->child_list, child_list)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2684

*parent = prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2713

svm_range_best_restore_location(struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2722

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2733

if (prange->preferred_loc == gpuid ||

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2734

prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2735

return prange->preferred_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2736

} else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2737

preferred_node = svm_range_get_node_by_id(prange, prange->preferred_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2739

return prange->preferred_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2743

if (test_bit(*gpuidx, prange->bitmap_access))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2746

if (test_bit(*gpuidx, prange->bitmap_aip)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2747

if (!prange->actual_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2750

bo_node = svm_range_get_node_by_id(prange, prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2752

return prange->actual_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

276

svm_range_dma_unmap_dev(dev, dma_addr, 0, prange->npages);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

280

static void svm_range_free(struct svm_range *prange, bool do_unmap)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

282

uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

283

struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

286

pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

287

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2871

struct svm_range *prange = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

289

svm_range_vram_node_free(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2896

prange = svm_range_new(&p->svms, start, last, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2897

if (!prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2903

svm_range_free(prange, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2908

prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

291

svm_range_dma_unmap(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2910

svm_range_add_to_svms(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2911

svm_range_add_notifier_locked(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2913

return prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2928

static bool svm_range_skip_recover(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2930

struct svm_range_list *svms = prange->svms;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2933

if (list_empty(&prange->deferred_list) &&

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2934

list_empty(&prange->child_list)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

294

pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2940

if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2942

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2945

if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP ||

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2946

prange->work_item.op == SVM_OP_ADD_RANGE) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

2948

svms, prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3002

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

301

if (prange->dma_addr[gpuidx]) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

302

kvfree(prange->dma_addr[gpuidx]);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

303

prange->dma_addr[gpuidx] = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

307

mutex_destroy(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

308

mutex_destroy(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3083

prange = svm_range_from_addr(svms, addr, NULL);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3084

if (!prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

309

kfree(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3098

prange = svm_range_create_unregistered_range(node, p, mm, addr);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3099

if (!prange) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3110

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3112

if (svm_range_skip_recover(prange)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3119

if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3122

svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3144

best_loc = svm_range_best_restore_location(prange, node, &gpuidx);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3147

svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3153

svms, prange->start, prange->last, best_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3154

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3160

size = 1UL << prange->granularity;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3161

start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3162

last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3163

if (prange->actual_loc != 0 || best_loc != 0) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3165

r = svm_migrate_to_vram(prange, best_loc, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3173

if (prange->actual_loc && prange->actual_loc != best_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3174

r = svm_migrate_vram_to_ram(prange, mm, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3180

r = svm_migrate_vram_to_ram(prange, mm, start, last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3192

r = svm_range_validate_and_map(mm, start, last, prange, gpuidx, false,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3203

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3226

struct svm_range *prange, *pchild;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3235

list_for_each_entry(prange, &p->svms.list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3236

svm_range_lock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3237

list_for_each_entry(pchild, &prange->child_list, child_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3251

size = (prange->last - prange->start + 1) << PAGE_SHIFT;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3263

svm_range_unlock(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3283

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

329

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3302

list_for_each_entry_safe(prange, next, &p->svms.list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3303

svm_range_unlink(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3304

svm_range_remove_notifier(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3305

svm_range_free(prange, true);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

332

prange = kzalloc(sizeof(*prange), GFP_KERNEL);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

333

if (!prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

341

kfree(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

344

prange->npages = size;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

345

prange->svms = svms;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

346

prange->start = start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3463

svm_range_best_prefetch_location(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3466

uint32_t best_loc = prange->prefetch_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

347

prange->last = last;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3472

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3477

bo_node = svm_range_get_node_by_id(prange, best_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

348

INIT_LIST_HEAD(&prange->list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

349

INIT_LIST_HEAD(&prange->update_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3490

bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3492

bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

350

INIT_LIST_HEAD(&prange->svm_bo_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

351

INIT_LIST_HEAD(&prange->deferred_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3513

p->xnack_enabled, &p->svms, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

352

INIT_LIST_HEAD(&prange->child_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

353

atomic_set(&prange->invalid, 0);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

354

prange->validate_timestamp = 0;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3544

svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

355

prange->vram_pages = 0;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3551

best_loc = svm_range_best_prefetch_location(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3559

(best_loc == 0 && prange->actual_loc == 0))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

356

mutex_init(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3563

r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3569

r = svm_migrate_to_vram(prange, best_loc, prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

357

mutex_init(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

360

bitmap_copy(prange->bitmap_access, svms->bitmap_supported,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3611

struct svm_range *prange =

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3616

list_del_init(&prange->svm_bo_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3619

pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3620

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3622

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3627

r = svm_migrate_vram_to_ram(prange, mm,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3628

prange->start, prange->last,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

363

svm_range_set_default_attributes(svms, &prange->preferred_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3630

} while (!r && prange->actual_loc && --retries);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3632

if (!r && prange->actual_loc)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3635

if (!prange->actual_loc) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3636

mutex_lock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3637

prange->svm_bo = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3638

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

364

&prange->prefetch_loc,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3640

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

365

&prange->granularity, &prange->flags);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3668

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

369

return prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3708

list_for_each_entry_safe(prange, next, &insert_list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3709

svm_range_add_to_svms(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3710

svm_range_add_notifier_locked(mm, prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3712

list_for_each_entry(prange, &update_list, update_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3713

svm_range_apply_attrs(p, prange, nattr, attrs, &update_mapping);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3718

list_for_each_entry_safe(prange, next, &remove_list, update_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3720

prange->svms, prange, prange->start,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3721

prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3722

svm_range_unlink(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3723

svm_range_remove_notifier(prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3724

svm_range_free(prange, false);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3733

list_for_each_entry(prange, &update_list, update_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3736

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3738

r = svm_range_trigger_migration(mm, prange, &migrated);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3743

(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3744

prange->mapped_to_gpu) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3746

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3751

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3755

flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3757

r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3763

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3768

list_for_each_entry(prange, &remap_list, update_list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3770

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3771

mutex_lock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3772

r = svm_range_validate_and_map(mm, prange->start, prange->last, prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3773

MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3776

mutex_unlock(&prange->migrate_mutex);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3810

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3889

prange = container_of(node, struct svm_range, it_node);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

389

struct svm_range *prange =

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3893

if (prange->preferred_loc ==

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3896

location != prange->preferred_loc)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3900

location = prange->preferred_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3904

if (prange->prefetch_loc ==

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3907

prefetch_loc != prange->prefetch_loc)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3911

prefetch_loc = prange->prefetch_loc;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3916

prange->bitmap_access, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3918

prange->bitmap_aip, MAX_GPU_INSTANCE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3921

flags_and &= prange->flags;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3922

flags_or |= prange->flags;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3925

if (get_granularity && prange->granularity < granularity)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

3926

granularity = prange->granularity;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

395

list_del_init(&prange->svm_bo_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

398

pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

399

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

400

mutex_lock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

401

prange->svm_bo = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

403

WARN_ONCE(prange->actual_loc, "prange should not hold vram page");

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

404

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4135

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4143

list_for_each_entry(prange, &svms->list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4145

prange, prange->start, prange->npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4146

prange->start + prange->npages - 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4194

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4236

list_for_each_entry(prange, &svms->list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4239

svm_priv->start_addr = prange->start;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4240

svm_priv->size = prange->npages;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4243

prange, prange->start, prange->npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4244

prange->start + prange->npages - 1,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

4245

prange->npages * PAGE_SIZE);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

468

svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

470

mutex_lock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

471

if (!prange->svm_bo) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

472

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

475

if (prange->ttm_res) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

477

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

480

if (svm_bo_ref_unless_zero(prange->svm_bo)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

486

if (prange->svm_bo->node != node) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

487

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

489

spin_lock(&prange->svm_bo->list_lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

490

list_del_init(&prange->svm_bo_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

491

spin_unlock(&prange->svm_bo->list_lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

493

svm_range_bo_unref(prange->svm_bo);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

496

if (READ_ONCE(prange->svm_bo->evicting)) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

502

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

503

svm_bo = prange->svm_bo;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

505

svm_range_bo_unref(prange->svm_bo);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

515

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

517

prange->svms, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

519

prange->ttm_res = prange->svm_bo->bo->tbo.resource;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

524

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

532

while (!list_empty_careful(&prange->svm_bo_list) || prange->svm_bo)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

554

svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

566

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

568

p->lead_thread->pid, prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

569

prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

571

if (svm_range_validate_svm_bo(node, prange))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

594

bp.size = prange->npages * PAGE_SIZE;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

642

prange->svm_bo = svm_bo;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

643

prange->ttm_res = bo->tbo.resource;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

644

prange->offset = 0;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

647

list_add(&prange->svm_bo_list, &svm_bo->range_list);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

650

pdd = svm_range_get_pdd_by_node(prange, node);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

661

prange->ttm_res = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

666

void svm_range_vram_node_free(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

669

mutex_lock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

671

if (prange->ttm_res) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

672

prange->ttm_res = NULL;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

673

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

674

svm_range_bo_unref(prange->svm_bo);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

676

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

680

svm_range_get_node_by_id(struct svm_range *prange, uint32_t gpu_id)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

685

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

696

svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

700

p = container_of(prange->svms, struct kfd_process, svms);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

764

svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

774

prange->preferred_loc = attrs[i].value;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

777

prange->prefetch_loc = attrs[i].value;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

788

bitmap_clear(prange->bitmap_access, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

789

bitmap_clear(prange->bitmap_aip, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

791

bitmap_set(prange->bitmap_access, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

792

bitmap_clear(prange->bitmap_aip, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

794

bitmap_clear(prange->bitmap_access, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

795

bitmap_set(prange->bitmap_aip, gpuidx, 1);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

800

prange->flags |= attrs[i].value;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

804

prange->flags &= ~attrs[i].value;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

807

prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

816

svm_range_is_same_attrs(struct kfd_process *p, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

825

if (prange->preferred_loc != attrs[i].value)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

839

if (test_bit(gpuidx, prange->bitmap_access) ||

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

840

test_bit(gpuidx, prange->bitmap_aip))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

843

if (!test_bit(gpuidx, prange->bitmap_access))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

846

if (!test_bit(gpuidx, prange->bitmap_aip))

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

851

if ((prange->flags & attrs[i].value) != attrs[i].value)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

855

if ((prange->flags & attrs[i].value) != 0)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

859

if (prange->granularity != attrs[i].value)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

882

struct svm_range *prange;

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

887

list_for_each_entry(prange, &svms->list, list) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

889

prange, prange->start, prange->npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

890

prange->start + prange->npages - 1,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

891

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

898

prange = container_of(node, struct svm_range, it_node);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

900

prange, prange->start, prange->npages,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

901

prange->start + prange->npages - 1,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

902

prange->actual_loc);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

93

static void svm_range_unlink(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

95

pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

96

prange, prange->start, prange->last);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

98

if (prange->svm_bo) {

sys/dev/pci/drm/amd/amdkfd/kfd_svm.c

99

spin_lock(&prange->svm_bo->list_lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

143

static inline void svm_range_lock(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

145

mutex_lock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

146

prange->saved_flags = memalloc_noreclaim_save();

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

149

static inline void svm_range_unlock(struct svm_range *prange)

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

151

memalloc_noreclaim_restore(prange->saved_flags);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

152

mutex_unlock(&prange->lock);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

171

struct kfd_node *svm_range_get_node_by_id(struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

173

int svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

175

void svm_range_vram_node_free(struct svm_range *prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

181

struct svm_range *prange, struct mm_struct *mm,

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

186

void svm_range_dma_unmap(struct svm_range *prange);

sys/dev/pci/drm/amd/amdkfd/kfd_svm.h

198

svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node);

usr.bin/cvs/diff3.c

144

static void prange(struct range *);

usr.bin/cvs/diff3.c

665

prange(rold);

usr.bin/cvs/diff3.c

798

prange(&de[n].old);

usr.bin/diff3/diff3prog.c

135

void prange(struct range *);

usr.bin/diff3/diff3prog.c

416

prange(rold);

usr.bin/diff3/diff3prog.c

555

prange(&de[n].old);

usr.bin/rcs/diff3.c

143

static void prange(struct range *);

usr.bin/rcs/diff3.c

760

prange(rold);

usr.bin/rcs/diff3.c

893

prange(&de[n].old);