/* * Copyright 2016-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-fence.h> #include <linux/spinlock.h> #include <linux/atomic.h> #include <linux/stacktrace.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/sched/mm.h> #include "amdgpu_amdkfd.h" #include "kfd_svm.h" static const struct dma_fence_ops amdkfd_fence_ops; static atomic_t fence_seq = ATOMIC_INIT(0); /* Eviction Fence * Fence helper functions to deal with KFD memory eviction. * Big Idea - Since KFD submissions are done by user queues, a BO cannot be * evicted unless all the user queues for that process are evicted. * * All the BOs in a process share an eviction fence. When process X wants * to map VRAM memory but TTM can't find enough space, TTM will attempt to * evict BOs from its LRU list. TTM checks if the BO is valuable to evict * by calling ttm_device_funcs->eviction_valuable(). * * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs * to process X. Otherwise, it will return true to indicate BO can be * evicted by TTM. * * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler. * * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to * nofity when the BO is free to move. fence_add_callback --> enable_signaling * --> amdgpu_amdkfd_fence.enable_signaling * * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce * user queues and signal fence. The work item will also start another delayed * work item to restore BOs */ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, struct mm_struct *mm, struct svm_range_bo *svm_bo, u16 context_id) { struct amdgpu_amdkfd_fence *fence; fence = kzalloc_obj(*fence); if (fence == NULL) return NULL; /* This reference gets released in amdkfd_fence_release */ mmgrab(mm); fence->mm = mm; get_task_comm(fence->timeline_name, current); spin_lock_init(&fence->lock); fence->svm_bo = svm_bo; fence->context_id = context_id; dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock, context, atomic_inc_return(&fence_seq)); return fence; } struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) { struct amdgpu_amdkfd_fence *fence; if (!f) return NULL; fence = container_of(f, struct amdgpu_amdkfd_fence, base); if (f->ops == &amdkfd_fence_ops) return fence; return NULL; } static const char *amdkfd_fence_get_driver_name(struct dma_fence *f) { return "amdgpu_amdkfd_fence"; } static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f) { struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); return fence ? fence->timeline_name : NULL; } /** * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict * a KFD BO and schedules a job to move the BO. * If fence is already signaled return true. * If fence is not signaled schedule a evict KFD process work item. * * @f: dma_fence */ static bool amdkfd_fence_enable_signaling(struct dma_fence *f) { struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); if (!fence) return false; if (dma_fence_is_signaled(f)) return true; /* if fence->svm_bo is NULL, means this fence is created through * init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos(). * Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence. */ if (!fence->svm_bo) { if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f)) return true; } else { if (!svm_range_schedule_evict_svm_bo(fence)) return true; } return false; } /** * amdkfd_fence_release - callback that fence can be freed * * @f: dma_fence * * This function is called when the reference count becomes zero. * Drops the mm_struct reference and RCU schedules freeing up the fence. */ static void amdkfd_fence_release(struct dma_fence *f) { struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); /* Unconditionally signal the fence. The process is getting * terminated. */ if (WARN_ON(!fence)) return; /* Not an amdgpu_amdkfd_fence */ mmdrop(fence->mm); kfree_rcu(f, rcu); } /** * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm * * @f: [IN] fence * @mm: [IN] mm that needs to be verified * * Check if @mm is same as that of the fence @f, if same return TRUE else * return FALSE. * For svm bo, which support vram overcommitment, always return FALSE. */ bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) { struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); if (!fence) return false; else if (fence->mm == mm && !fence->svm_bo) return true; return false; } static const struct dma_fence_ops amdkfd_fence_ops = { .get_driver_name = amdkfd_fence_get_driver_name, .get_timeline_name = amdkfd_fence_get_timeline_name, .enable_signaling = amdkfd_fence_enable_signaling, .release = amdkfd_fence_release, };
