// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_ggtt.h"

#include <kunit/visibility.h>
#include <linux/fault-inject.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/sizes.h>

#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/intel/i915_drm.h>
#include <generated/xe_wa_oob.h>

#include "regs/xe_gt_regs.h"
#include "regs/xe_gtt_defs.h"
#include "regs/xe_regs.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_gt_printk.h"
#include "xe_gt_types.h"
#include "xe_map.h"
#include "xe_mmio.h"
#include "xe_pat.h"
#include "xe_pm.h"
#include "xe_res_cursor.h"
#include "xe_sriov.h"
#include "xe_tile_printk.h"
#include "xe_tile_sriov_vf.h"
#include "xe_tlb_inval.h"
#include "xe_wa.h"
#include "xe_wopcm.h"

/**
 * DOC: Global Graphics Translation Table (GGTT)
 *
 * Xe GGTT implements the support for a Global Virtual Address space that is used
 * for resources that are accessible to privileged (i.e. kernel-mode) processes,
 * and not tied to a specific user-level process. For example, the Graphics
 * micro-Controller (GuC) and Display Engine (if present) utilize this Global
 * address space.
 *
 * The Global GTT (GGTT) translates from the Global virtual address to a physical
 * address that can be accessed by HW. The GGTT is a flat, single-level table.
 *
 * Xe implements a simplified version of the GGTT specifically managing only a
 * certain range of it that goes from the Write Once Protected Content Memory (WOPCM)
 * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to
 * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space
 * is limited on both ends of the GGTT, because the GuC shim HW redirects
 * accesses to those addresses to other HW areas instead of going through the
 * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size,
 * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things
 * simple, instead of checking each object to see if they are accessed by GuC or
 * not, we just exclude those areas from the allocator. Additionally, to simplify
 * the driver load, we use the maximum WOPCM size in this logic instead of the
 * programmed one, so we don't need to wait until the actual size to be
 * programmed is determined (which requires FW fetch) before initializing the
 * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs
 * depending on the platform) but we can live with this. Another benefit of this
 * is the GuC bootrom can't access anything below the WOPCM max size so anything
 * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT
 * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max
 * give us the correct placement for free.
 */

#define XE_GGTT_FLAGS_64K       BIT(0)
#define XE_GGTT_FLAGS_ONLINE    BIT(1)

/**
 * struct xe_ggtt_node - A node in GGTT.
 *
 * This struct is allocated with xe_ggtt_insert_node(,_transform) or xe_ggtt_insert_bo(,_at).
 * It will be deallocated using xe_ggtt_node_remove().
 */
struct xe_ggtt_node {
        /** @ggtt: Back pointer to xe_ggtt where this region will be inserted at */
        struct xe_ggtt *ggtt;
        /** @base: A drm_mm_node */
        struct drm_mm_node base;
        /** @delayed_removal_work: The work struct for the delayed removal */
        struct work_struct delayed_removal_work;
        /** @invalidate_on_remove: If it needs invalidation upon removal */
        bool invalidate_on_remove;
};

/**
 * struct xe_ggtt_pt_ops - GGTT Page table operations
 * Which can vary from platform to platform.
 */
struct xe_ggtt_pt_ops {
        /** @pte_encode_flags: Encode PTE flags for a given BO */
        u64 (*pte_encode_flags)(struct xe_bo *bo, u16 pat_index);

        /** @ggtt_set_pte: Directly write into GGTT's PTE */
        xe_ggtt_set_pte_fn ggtt_set_pte;

        /** @ggtt_get_pte: Directly read from GGTT's PTE */
        u64 (*ggtt_get_pte)(struct xe_ggtt *ggtt, u64 addr);
};

/**
 * struct xe_ggtt - Main GGTT struct
 *
 * In general, each tile can contains its own Global Graphics Translation Table
 * (GGTT) instance.
 */
struct xe_ggtt {
        /** @tile: Back pointer to tile where this GGTT belongs */
        struct xe_tile *tile;
        /** @start: Start offset of GGTT */
        u64 start;
        /** @size: Total usable size of this GGTT */
        u64 size;

        /**
         * @flags: Flags for this GGTT
         * Acceptable flags:
         * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
         * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock
         *   after init
         */
        unsigned int flags;
        /** @scratch: Internal object allocation used as a scratch page */
        struct xe_bo *scratch;
        /** @lock: Mutex lock to protect GGTT data */
        struct mutex lock;
        /**
         *  @gsm: The iomem pointer to the actual location of the translation
         * table located in the GSM for easy PTE manipulation
         */
        u64 __iomem *gsm;
        /** @pt_ops: Page Table operations per platform */
        const struct xe_ggtt_pt_ops *pt_ops;
        /** @mm: The memory manager used to manage individual GGTT allocations */
        struct drm_mm mm;
        /** @access_count: counts GGTT writes */
        unsigned int access_count;
        /** @wq: Dedicated unordered work queue to process node removals */
        struct workqueue_struct *wq;
};

static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
{
        u64 pte = XE_PAGE_PRESENT;

        if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
                pte |= XE_GGTT_PTE_DM;

        return pte;
}

static u64 xelpg_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
{
        struct xe_device *xe = xe_bo_device(bo);
        u64 pte;

        pte = xelp_ggtt_pte_flags(bo, pat_index);

        xe_assert(xe, pat_index <= 3);

        if (pat_index & BIT(0))
                pte |= XELPG_GGTT_PTE_PAT0;

        if (pat_index & BIT(1))
                pte |= XELPG_GGTT_PTE_PAT1;

        return pte;
}

static unsigned int probe_gsm_size(struct pci_dev *pdev)
{
        u16 gmch_ctl, ggms;

        pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
        ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
        return ggms ? SZ_1M << ggms : 0;
}

static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
{
        struct xe_tile *tile = ggtt->tile;
        struct xe_gt *affected_gt;
        u32 max_gtt_writes;

        if (tile->primary_gt && XE_GT_WA(tile->primary_gt, 22019338487)) {
                affected_gt = tile->primary_gt;
                max_gtt_writes = 1100;

                /* Only expected to apply to primary GT on dgpu platforms */
                xe_tile_assert(tile, IS_DGFX(tile_to_xe(tile)));
        } else {
                affected_gt = tile->media_gt;
                max_gtt_writes = 63;

                /* Only expected to apply to media GT on igpu platforms */
                xe_tile_assert(tile, !IS_DGFX(tile_to_xe(tile)));
        }

        /*
         * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
         * to wait for completion of prior GTT writes before letting this through.
         * This needs to be done for all GGTT writes originating from the CPU.
         */
        lockdep_assert_held(&ggtt->lock);

        if ((++ggtt->access_count % max_gtt_writes) == 0) {
                xe_mmio_write32(&affected_gt->mmio, GMD_ID, 0x0);
                ggtt->access_count = 0;
        }
}

/**
 * xe_ggtt_start - Get starting offset of GGTT.
 * @ggtt: &xe_ggtt
 *
 * Returns: Starting offset for this &xe_ggtt.
 */
u64 xe_ggtt_start(struct xe_ggtt *ggtt)
{
        return ggtt->start;
}

/**
 * xe_ggtt_size - Get size of GGTT.
 * @ggtt: &xe_ggtt
 *
 * Returns: Total usable size of this &xe_ggtt.
 */
u64 xe_ggtt_size(struct xe_ggtt *ggtt)
{
        return ggtt->size;
}

static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
{
        xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
        xe_tile_assert(ggtt->tile, addr < ggtt->start + ggtt->size);

        writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
}

static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
{
        xe_ggtt_set_pte(ggtt, addr, pte);
        ggtt_update_access_counter(ggtt);
}

static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr)
{
        xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
        xe_tile_assert(ggtt->tile, addr < ggtt->start + ggtt->size);

        return readq(&ggtt->gsm[addr >> XE_PTE_SHIFT]);
}

static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
{
        u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), XE_CACHE_WB);
        u64 end = start + size - 1;
        u64 scratch_pte;

        xe_tile_assert(ggtt->tile, start < end);

        if (ggtt->scratch)
                scratch_pte = xe_bo_addr(ggtt->scratch, 0, XE_PAGE_SIZE) |
                              ggtt->pt_ops->pte_encode_flags(ggtt->scratch,
                                                             pat_index);
        else
                scratch_pte = 0;

        while (start < end) {
                ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
                start += XE_PAGE_SIZE;
        }
}

static void primelockdep(struct xe_ggtt *ggtt)
{
        if (!IS_ENABLED(CONFIG_LOCKDEP))
                return;

        fs_reclaim_acquire(GFP_KERNEL);
        might_lock(&ggtt->lock);
        fs_reclaim_release(GFP_KERNEL);
}

/**
 * xe_ggtt_alloc - Allocate a GGTT for a given &xe_tile
 * @tile: &xe_tile
 *
 * Allocates a &xe_ggtt for a given tile.
 *
 * Return: &xe_ggtt on success, or NULL when out of memory.
 */
struct xe_ggtt *xe_ggtt_alloc(struct xe_tile *tile)
{
        struct xe_device *xe = tile_to_xe(tile);
        struct xe_ggtt *ggtt;

        ggtt = drmm_kzalloc(&xe->drm, sizeof(*ggtt), GFP_KERNEL);
        if (!ggtt)
                return NULL;

        if (drmm_mutex_init(&xe->drm, &ggtt->lock))
                return NULL;

        primelockdep(ggtt);
        ggtt->tile = tile;

        return ggtt;
}

static void ggtt_fini_early(struct drm_device *drm, void *arg)
{
        struct xe_ggtt *ggtt = arg;

        destroy_workqueue(ggtt->wq);
        drm_mm_takedown(&ggtt->mm);
}

static void ggtt_fini(void *arg)
{
        struct xe_ggtt *ggtt = arg;

        ggtt->scratch = NULL;
}

#ifdef CONFIG_LOCKDEP
void xe_ggtt_might_lock(struct xe_ggtt *ggtt)
{
        might_lock(&ggtt->lock);
}
#endif

static const struct xe_ggtt_pt_ops xelp_pt_ops = {
        .pte_encode_flags = xelp_ggtt_pte_flags,
        .ggtt_set_pte = xe_ggtt_set_pte,
        .ggtt_get_pte = xe_ggtt_get_pte,
};

static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
        .pte_encode_flags = xelpg_ggtt_pte_flags,
        .ggtt_set_pte = xe_ggtt_set_pte,
        .ggtt_get_pte = xe_ggtt_get_pte,
};

static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
        .pte_encode_flags = xelpg_ggtt_pte_flags,
        .ggtt_set_pte = xe_ggtt_set_pte_and_flush,
        .ggtt_get_pte = xe_ggtt_get_pte,
};

static void __xe_ggtt_init_early(struct xe_ggtt *ggtt, u64 start, u64 size)
{
        ggtt->start = start;
        ggtt->size = size;
        drm_mm_init(&ggtt->mm, 0, size);
}

int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 start, u32 size)
{
        __xe_ggtt_init_early(ggtt, start, size);
        return 0;
}
EXPORT_SYMBOL_IF_KUNIT(xe_ggtt_init_kunit);

static void dev_fini_ggtt(void *arg)
{
        struct xe_ggtt *ggtt = arg;

        scoped_guard(mutex, &ggtt->lock)
                ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE;
        drain_workqueue(ggtt->wq);
}

/**
 * xe_ggtt_init_early - Early GGTT initialization
 * @ggtt: the &xe_ggtt to be initialized
 *
 * It allows to create new mappings usable by the GuC.
 * Mappings are not usable by the HW engines, as it doesn't have scratch nor
 * initial clear done to it yet. That will happen in the regular, non-early
 * GGTT initialization.
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_init_early(struct xe_ggtt *ggtt)
{
        struct xe_device *xe = tile_to_xe(ggtt->tile);
        struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
        unsigned int gsm_size;
        u64 ggtt_start, wopcm = xe_wopcm_size(xe), ggtt_size;
        int err;

        if (!IS_SRIOV_VF(xe)) {
                if (GRAPHICS_VERx100(xe) >= 1250)
                        gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
                else
                        gsm_size = probe_gsm_size(pdev);
                if (gsm_size == 0) {
                        xe_tile_err(ggtt->tile, "Hardware reported no preallocated GSM\n");
                        return -ENOMEM;
                }
                ggtt_start = wopcm;
                ggtt_size = (gsm_size / 8) * (u64)XE_PAGE_SIZE - ggtt_start;
        } else {
                ggtt_start = xe_tile_sriov_vf_ggtt_base(ggtt->tile);
                ggtt_size = xe_tile_sriov_vf_ggtt(ggtt->tile);

                if (ggtt_start < wopcm ||
                    ggtt_start + ggtt_size > GUC_GGTT_TOP) {
                        xe_tile_err(ggtt->tile, "Invalid GGTT configuration: %#llx-%#llx\n",
                                    ggtt_start, ggtt_start + ggtt_size - 1);
                        return -ERANGE;
                }
        }

        ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
        if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
                ggtt->flags |= XE_GGTT_FLAGS_64K;

        if (ggtt_size + ggtt_start > GUC_GGTT_TOP)
                ggtt_size = GUC_GGTT_TOP - ggtt_start;

        if (GRAPHICS_VERx100(xe) >= 1270)
                ggtt->pt_ops =
                        (ggtt->tile->media_gt && XE_GT_WA(ggtt->tile->media_gt, 22019338487)) ||
                        (ggtt->tile->primary_gt && XE_GT_WA(ggtt->tile->primary_gt, 22019338487)) ?
                        &xelpg_pt_wa_ops : &xelpg_pt_ops;
        else
                ggtt->pt_ops = &xelp_pt_ops;

        ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM | WQ_PERCPU, 0);
        if (!ggtt->wq)
                return -ENOMEM;

        __xe_ggtt_init_early(ggtt, ggtt_start, ggtt_size);

        err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
        if (err)
                return err;

        ggtt->flags |= XE_GGTT_FLAGS_ONLINE;
        return devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
}
ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */

static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);

static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
{
        struct drm_mm_node *hole;
        u64 start, end;

        /* Display may have allocated inside ggtt, so be careful with clearing here */
        mutex_lock(&ggtt->lock);
        drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
                xe_ggtt_clear(ggtt, ggtt->start + start, end - start);

        xe_ggtt_invalidate(ggtt);
        mutex_unlock(&ggtt->lock);
}

static void ggtt_node_fini(struct xe_ggtt_node *node)
{
        kfree(node);
}

static void ggtt_node_remove(struct xe_ggtt_node *node)
{
        struct xe_ggtt *ggtt = node->ggtt;
        bool bound;

        mutex_lock(&ggtt->lock);
        bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE;
        if (bound)
                xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node));
        drm_mm_remove_node(&node->base);
        node->base.size = 0;
        if (bound && node->invalidate_on_remove)
                xe_ggtt_invalidate(ggtt);
        mutex_unlock(&ggtt->lock);

        ggtt_node_fini(node);
}

static void ggtt_node_remove_work_func(struct work_struct *work)
{
        struct xe_ggtt_node *node = container_of(work, typeof(*node),
                                                 delayed_removal_work);
        struct xe_device *xe = tile_to_xe(node->ggtt->tile);

        guard(xe_pm_runtime)(xe);
        ggtt_node_remove(node);
}

/**
 * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT
 * @node: the &xe_ggtt_node to be removed
 * @invalidate: if node needs invalidation upon removal
 */
void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
{
        struct xe_ggtt *ggtt;
        struct xe_device *xe;

        if (!node || !node->ggtt)
                return;

        ggtt = node->ggtt;
        xe = tile_to_xe(ggtt->tile);

        node->invalidate_on_remove = invalidate;

        if (xe_pm_runtime_get_if_active(xe)) {
                ggtt_node_remove(node);
                xe_pm_runtime_put(xe);
        } else {
                queue_work(ggtt->wq, &node->delayed_removal_work);
        }
}

/**
 * xe_ggtt_init - Regular non-early GGTT initialization
 * @ggtt: the &xe_ggtt to be initialized
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_init(struct xe_ggtt *ggtt)
{
        struct xe_device *xe = tile_to_xe(ggtt->tile);
        unsigned int flags;
        int err;

        /*
         * So we don't need to worry about 64K GGTT layout when dealing with
         * scratch entries, rather keep the scratch page in system memory on
         * platforms where 64K pages are needed for VRAM.
         */
        flags = 0;
        if (ggtt->flags & XE_GGTT_FLAGS_64K)
                flags |= XE_BO_FLAG_SYSTEM;
        else
                flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);

        ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
        if (IS_ERR(ggtt->scratch)) {
                err = PTR_ERR(ggtt->scratch);
                goto err;
        }

        xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, xe_bo_size(ggtt->scratch));

        xe_ggtt_initial_clear(ggtt);

        return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt);
err:
        ggtt->scratch = NULL;
        return err;
}

static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
{
        int err;

        if (!gt)
                return;

        err = xe_tlb_inval_ggtt(&gt->tlb_inval);
        xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err));
}

static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
{
        struct xe_device *xe = tile_to_xe(ggtt->tile);

        /*
         * XXX: Barrier for GGTT pages. Unsure exactly why this required but
         * without this LNL is having issues with the GuC reading scratch page
         * vs. correct GGTT page. Not particularly a hot code path so blindly
         * do a mmio read here which results in GuC reading correct GGTT page.
         */
        xe_mmio_read32(xe_root_tile_mmio(xe), VF_CAP_REG);

        /* Each GT in a tile has its own TLB to cache GGTT lookups */
        ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
        ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
}

/**
 * xe_ggtt_shift_nodes() - Shift GGTT nodes to adjust for a change in usable address range.
 * @ggtt: the &xe_ggtt struct instance
 * @new_start: new location of area provisioned for current VF
 *
 * Ensure that all struct &xe_ggtt_node are moved to the @new_start base address
 * by changing the base offset of the GGTT.
 *
 * This function may be called multiple times during recovery, but if
 * @new_start is unchanged from the current base, it's a noop.
 *
 * @new_start should be a value between xe_wopcm_size() and #GUC_GGTT_TOP.
 */
void xe_ggtt_shift_nodes(struct xe_ggtt *ggtt, u64 new_start)
{
        guard(mutex)(&ggtt->lock);

        xe_tile_assert(ggtt->tile, new_start >= xe_wopcm_size(tile_to_xe(ggtt->tile)));
        xe_tile_assert(ggtt->tile, new_start + ggtt->size <= GUC_GGTT_TOP);

        /* pairs with READ_ONCE in xe_ggtt_node_addr() */
        WRITE_ONCE(ggtt->start, new_start);
}

static int xe_ggtt_insert_node_locked(struct xe_ggtt_node *node,
                                      u32 size, u32 align, u32 mm_flags)
{
        return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
                                          mm_flags);
}

static struct xe_ggtt_node *ggtt_node_init(struct xe_ggtt *ggtt)
{
        struct xe_ggtt_node *node = kzalloc_obj(*node, GFP_NOFS);

        if (!node)
                return ERR_PTR(-ENOMEM);

        INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func);
        node->ggtt = ggtt;

        return node;
}

/**
 * xe_ggtt_insert_node - Insert a &xe_ggtt_node into the GGTT
 * @ggtt: the &xe_ggtt into which the node should be inserted.
 * @size: size of the node
 * @align: alignment constrain of the node
 *
 * Return: &xe_ggtt_node on success or a ERR_PTR on failure.
 */
struct xe_ggtt_node *xe_ggtt_insert_node(struct xe_ggtt *ggtt, u32 size, u32 align)
{
        struct xe_ggtt_node *node;
        int ret;

        node = ggtt_node_init(ggtt);
        if (IS_ERR(node))
                return node;

        guard(mutex)(&ggtt->lock);
        ret = xe_ggtt_insert_node_locked(node, size, align,
                                         DRM_MM_INSERT_HIGH);
        if (ret) {
                ggtt_node_fini(node);
                return ERR_PTR(ret);
        }

        return node;
}

/**
 * xe_ggtt_node_pt_size() - Get the size of page table entries needed to map a GGTT node.
 * @node: the &xe_ggtt_node
 *
 * Return: GGTT node page table entries size in bytes.
 */
size_t xe_ggtt_node_pt_size(const struct xe_ggtt_node *node)
{
        if (!node)
                return 0;

        return node->base.size / XE_PAGE_SIZE * sizeof(u64);
}

/**
 * xe_ggtt_map_bo - Map the BO into GGTT
 * @ggtt: the &xe_ggtt where node will be mapped
 * @node: the &xe_ggtt_node where this BO is mapped
 * @bo: the &xe_bo to be mapped
 * @pte: The pte flags to append.
 */
static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node,
                           struct xe_bo *bo, u64 pte)
{
        u64 start, end;
        struct xe_res_cursor cur;

        if (XE_WARN_ON(!node))
                return;

        start = xe_ggtt_node_addr(node);
        end = start + xe_bo_size(bo);

        if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
                xe_assert(xe_bo_device(bo), bo->ttm.ttm);

                for (xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &cur);
                     cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE))
                        ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining,
                                                   pte | xe_res_dma(&cur));
        } else {
                /* Prepend GPU offset */
                pte |= vram_region_gpu_offset(bo->ttm.resource);

                for (xe_res_first(bo->ttm.resource, 0, xe_bo_size(bo), &cur);
                     cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE))
                        ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining,
                                                   pte + cur.start);
        }
}

/**
 * xe_ggtt_map_bo_unlocked - Restore a mapping of a BO into GGTT
 * @ggtt: the &xe_ggtt where node will be mapped
 * @bo: the &xe_bo to be mapped
 *
 * This is used to restore a GGTT mapping after suspend.
 */
void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
        u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
        u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode);
        u64 pte;

        mutex_lock(&ggtt->lock);
        pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index);
        xe_ggtt_map_bo(ggtt, bo->ggtt_node[ggtt->tile->id], bo, pte);
        mutex_unlock(&ggtt->lock);
}

/**
 * xe_ggtt_insert_node_transform - Insert a newly allocated &xe_ggtt_node into the GGTT
 * @ggtt: the &xe_ggtt where the node will inserted/reserved.
 * @bo: The bo to be transformed
 * @pte_flags: The extra GGTT flags to add to mapping.
 * @size: size of the node
 * @align: required alignment for node
 * @transform: transformation function that will populate the GGTT node, or NULL for linear mapping.
 * @arg: Extra argument to pass to the transformation function.
 *
 * This function allows inserting a GGTT node with a custom transformation function.
 * This is useful for display to allow inserting rotated framebuffers to GGTT.
 *
 * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
 */
struct xe_ggtt_node *xe_ggtt_insert_node_transform(struct xe_ggtt *ggtt,
                                                   struct xe_bo *bo, u64 pte_flags,
                                                   u64 size, u32 align,
                                                   xe_ggtt_transform_cb transform, void *arg)
{
        struct xe_ggtt_node *node;
        int ret;

        node = ggtt_node_init(ggtt);
        if (IS_ERR(node))
                return ERR_CAST(node);

        if (mutex_lock_interruptible(&ggtt->lock) < 0) {
                ret = -ERESTARTSYS;
                goto err;
        }

        ret = xe_ggtt_insert_node_locked(node, size, align, 0);
        if (ret)
                goto err_unlock;

        if (transform)
                transform(ggtt, node, pte_flags, ggtt->pt_ops->ggtt_set_pte, arg);
        else
                xe_ggtt_map_bo(ggtt, node, bo, pte_flags);

        mutex_unlock(&ggtt->lock);
        return node;

err_unlock:
        mutex_unlock(&ggtt->lock);
err:
        ggtt_node_fini(node);
        return ERR_PTR(ret);
}

static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
                                  u64 start, u64 end, struct drm_exec *exec)
{
        u64 alignment = bo->min_align > 0 ? bo->min_align : XE_PAGE_SIZE;
        u8 tile_id = ggtt->tile->id;
        int err;

        if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
                alignment = SZ_64K;

        if (XE_WARN_ON(bo->ggtt_node[tile_id])) {
                /* Someone's already inserted this BO in the GGTT */
                xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo));
                return 0;
        }

        err = xe_bo_validate(bo, NULL, false, exec);
        if (err)
                return err;

        xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));

        bo->ggtt_node[tile_id] = ggtt_node_init(ggtt);
        if (IS_ERR(bo->ggtt_node[tile_id])) {
                err = PTR_ERR(bo->ggtt_node[tile_id]);
                bo->ggtt_node[tile_id] = NULL;
                goto out;
        }

        mutex_lock(&ggtt->lock);
        /*
         * When inheriting the initial framebuffer, the framebuffer is
         * physically located at VRAM address 0, and usually at GGTT address 0 too.
         *
         * The display code will ask for a GGTT allocation between end of BO and
         * remainder of GGTT, unaware that the start is reserved by WOPCM.
         */
        if (start >= ggtt->start)
                start -= ggtt->start;
        else
                start = 0;

        /* Should never happen, but since we handle start, fail graciously for end */
        if (end >= ggtt->start)
                end -= ggtt->start;
        else
                end = 0;

        xe_tile_assert(ggtt->tile, end >= start + xe_bo_size(bo));

        err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node[tile_id]->base,
                                          xe_bo_size(bo), alignment, 0, start, end, 0);
        if (err) {
                ggtt_node_fini(bo->ggtt_node[tile_id]);
                bo->ggtt_node[tile_id] = NULL;
        } else {
                u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
                u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode);
                u64 pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index);

                xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pte);
        }
        mutex_unlock(&ggtt->lock);

        if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
                xe_ggtt_invalidate(ggtt);

out:
        xe_pm_runtime_put(tile_to_xe(ggtt->tile));

        return err;
}

/**
 * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space
 * @ggtt: the &xe_ggtt where bo will be inserted
 * @bo: the &xe_bo to be inserted
 * @start: address where it will be inserted
 * @end: end of the range where it will be inserted
 * @exec: The drm_exec transaction to use for exhaustive eviction.
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
                         u64 start, u64 end, struct drm_exec *exec)
{
        return __xe_ggtt_insert_bo_at(ggtt, bo, start, end, exec);
}

/**
 * xe_ggtt_insert_bo - Insert BO into GGTT
 * @ggtt: the &xe_ggtt where bo will be inserted
 * @bo: the &xe_bo to be inserted
 * @exec: The drm_exec transaction to use for exhaustive eviction.
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo,
                      struct drm_exec *exec)
{
        return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX, exec);
}

/**
 * xe_ggtt_remove_bo - Remove a BO from the GGTT
 * @ggtt: the &xe_ggtt where node will be removed
 * @bo: the &xe_bo to be removed
 */
void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
{
        u8 tile_id = ggtt->tile->id;

        if (XE_WARN_ON(!bo->ggtt_node[tile_id]))
                return;

        /* This BO is not currently in the GGTT */
        xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo));

        xe_ggtt_node_remove(bo->ggtt_node[tile_id],
                            bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
}

/**
 * xe_ggtt_largest_hole - Largest GGTT hole
 * @ggtt: the &xe_ggtt that will be inspected
 * @alignment: minimum alignment
 * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare
 *
 * Return: size of the largest continuous GGTT region
 */
u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare)
{
        const struct drm_mm *mm = &ggtt->mm;
        const struct drm_mm_node *entry;
        u64 hole_start, hole_end, hole_size;
        u64 max_hole = 0;

        mutex_lock(&ggtt->lock);
        drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
                hole_start = max(hole_start, ggtt->start);
                hole_start = ALIGN(hole_start, alignment);
                hole_end = ALIGN_DOWN(hole_end, alignment);
                if (hole_start >= hole_end)
                        continue;
                hole_size = hole_end - hole_start;
                if (spare)
                        *spare -= min3(*spare, hole_size, max_hole);
                max_hole = max(max_hole, hole_size);
        }

        mutex_unlock(&ggtt->lock);

        return max_hole;
}

#ifdef CONFIG_PCI_IOV
static u64 xe_encode_vfid_pte(u16 vfid)
{
        return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
}

static void xe_ggtt_assign_locked(const struct xe_ggtt_node *node, u16 vfid)
{
        struct xe_ggtt *ggtt = node->ggtt;
        u64 start = xe_ggtt_node_addr(node);
        u64 size = xe_ggtt_node_size(node);
        u64 end = start + size - 1;
        u64 pte = xe_encode_vfid_pte(vfid);

        lockdep_assert_held(&ggtt->lock);

        while (start < end) {
                ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
                start += XE_PAGE_SIZE;
        }

        xe_ggtt_invalidate(ggtt);
}

/**
 * xe_ggtt_assign - assign a GGTT region to the VF
 * @node: the &xe_ggtt_node to update
 * @vfid: the VF identifier
 *
 * This function is used by the PF driver to assign a GGTT region to the VF.
 * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
 * platforms VFs can't modify that either.
 */
void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
{
        guard(mutex)(&node->ggtt->lock);
        xe_ggtt_assign_locked(node, vfid);
}

/**
 * xe_ggtt_node_save() - Save a &xe_ggtt_node to a buffer.
 * @node: the &xe_ggtt_node to be saved
 * @dst: destination buffer
 * @size: destination buffer size in bytes
 * @vfid: VF identifier
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_node_save(struct xe_ggtt_node *node, void *dst, size_t size, u16 vfid)
{
        struct xe_ggtt *ggtt;
        u64 start, end;
        u64 *buf = dst;
        u64 pte;

        if (!node)
                return -ENOENT;

        ggtt = node->ggtt;
        guard(mutex)(&ggtt->lock);

        if (xe_ggtt_node_pt_size(node) != size)
                return -EINVAL;

        start = xe_ggtt_node_addr(node);
        end = start + xe_ggtt_node_size(node) - 1;

        while (start < end) {
                pte = ggtt->pt_ops->ggtt_get_pte(ggtt, start);
                if (vfid != u64_get_bits(pte, GGTT_PTE_VFID))
                        return -EPERM;

                *buf++ = u64_replace_bits(pte, 0, GGTT_PTE_VFID);
                start += XE_PAGE_SIZE;
        }

        return 0;
}

/**
 * xe_ggtt_node_load() - Load a &xe_ggtt_node from a buffer.
 * @node: the &xe_ggtt_node to be loaded
 * @src: source buffer
 * @size: source buffer size in bytes
 * @vfid: VF identifier
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_node_load(struct xe_ggtt_node *node, const void *src, size_t size, u16 vfid)
{
        u64 vfid_pte = xe_encode_vfid_pte(vfid);
        const u64 *buf = src;
        struct xe_ggtt *ggtt;
        u64 start, end;

        if (!node)
                return -ENOENT;

        ggtt = node->ggtt;
        guard(mutex)(&ggtt->lock);

        if (xe_ggtt_node_pt_size(node) != size)
                return -EINVAL;

        start = xe_ggtt_node_addr(node);
        end = start + xe_ggtt_node_size(node) - 1;

        while (start < end) {
                vfid_pte = u64_replace_bits(*buf++, vfid, GGTT_PTE_VFID);
                ggtt->pt_ops->ggtt_set_pte(ggtt, start, vfid_pte);
                start += XE_PAGE_SIZE;
        }
        xe_ggtt_invalidate(ggtt);

        return 0;
}

#endif

/**
 * xe_ggtt_dump - Dump GGTT for debug
 * @ggtt: the &xe_ggtt to be dumped
 * @p: the &drm_mm_printer helper handle to be used to dump the information
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
{
        int err;

        err = mutex_lock_interruptible(&ggtt->lock);
        if (err)
                return err;

        drm_mm_print(&ggtt->mm, p);
        mutex_unlock(&ggtt->lock);
        return err;
}

/**
 * xe_ggtt_print_holes - Print holes
 * @ggtt: the &xe_ggtt to be inspected
 * @alignment: min alignment
 * @p: the &drm_printer
 *
 * Print GGTT ranges that are available and return total size available.
 *
 * Return: Total available size.
 */
u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p)
{
        const struct drm_mm *mm = &ggtt->mm;
        const struct drm_mm_node *entry;
        u64 hole_start, hole_end, hole_size;
        u64 total = 0;
        char buf[10];

        mutex_lock(&ggtt->lock);
        drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
                hole_start = max(hole_start, ggtt->start);
                hole_start = ALIGN(hole_start, alignment);
                hole_end = ALIGN_DOWN(hole_end, alignment);
                if (hole_start >= hole_end)
                        continue;
                hole_size = hole_end - hole_start;
                total += hole_size;

                string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
                drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
                           hole_start, hole_end - 1, buf);
        }

        mutex_unlock(&ggtt->lock);

        return total;
}

/**
 * xe_ggtt_encode_pte_flags - Get PTE encoding flags for BO
 * @ggtt: &xe_ggtt
 * @bo: &xe_bo
 * @pat_index: The pat_index for the PTE.
 *
 * This function returns the pte_flags for a given BO, without  address.
 * It's used for DPT to fill a GGTT mapped BO with a linear lookup table.
 */
u64 xe_ggtt_encode_pte_flags(struct xe_ggtt *ggtt,
                             struct xe_bo *bo, u16 pat_index)
{
        return ggtt->pt_ops->pte_encode_flags(bo, pat_index);
}

/**
 * xe_ggtt_read_pte - Read a PTE from the GGTT
 * @ggtt: &xe_ggtt
 * @offset: the offset for which the mapping should be read.
 *
 * Used by testcases, and by display reading out an inherited bios FB.
 */
u64 xe_ggtt_read_pte(struct xe_ggtt *ggtt, u64 offset)
{
        return ioread64(ggtt->gsm + (offset / XE_PAGE_SIZE));
}

/**
 * xe_ggtt_node_addr - Get @node offset in GGTT.
 * @node: &xe_ggtt_node
 *
 * Get the GGTT offset for allocated node.
 */
u64 xe_ggtt_node_addr(const struct xe_ggtt_node *node)
{
        /* pairs with WRITE_ONCE in xe_ggtt_shift_nodes() */
        return node->base.start + READ_ONCE(node->ggtt->start);
}

/**
 * xe_ggtt_node_size - Get @node allocation size.
 * @node: &xe_ggtt_node
 *
 * Get the allocated node's size.
 */
u64 xe_ggtt_node_size(const struct xe_ggtt_node *node)
{
        return node->base.size;
}