// SPDX-License-Identifier: MIT
/*
 * Copyright © 2023 Intel Corporation
 */

#include <drm/drm_managed.h>

#include "xe_device_types.h"
#include "xe_force_wake.h"
#include "xe_gt_stats.h"
#include "xe_gt_types.h"
#include "xe_guc_ct.h"
#include "xe_guc_tlb_inval.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_tlb_inval.h"
#include "xe_trace.h"

/**
 * DOC: Xe TLB invalidation
 *
 * Xe TLB invalidation is implemented in two layers. The first is the frontend
 * API, which provides an interface for TLB invalidations to the driver code.
 * The frontend handles seqno assignment, synchronization (fences), and the
 * timeout mechanism. The frontend is implemented via an embedded structure
 * xe_tlb_inval that includes a set of ops hooking into the backend. The backend
 * interacts with the hardware (or firmware) to perform the actual invalidation.
 */

#define FENCE_STACK_BIT         DMA_FENCE_FLAG_USER_BITS

static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
{
        if (WARN_ON_ONCE(!fence->tlb_inval))
                return;

        xe_pm_runtime_put(fence->tlb_inval->xe);
        fence->tlb_inval = NULL; /* fini() should be called once */
}

static void
xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
{
        struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
        bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);

        lockdep_assert_held(&fence->tlb_inval->pending_lock);

        list_del(&fence->link);
        if (list_empty(&tlb_inval->pending_fences))
                cancel_delayed_work(&tlb_inval->fence_tdr);
        trace_xe_tlb_inval_fence_signal(fence->tlb_inval->xe, fence);
        xe_tlb_inval_fence_fini(fence);
        dma_fence_signal(&fence->base);
        if (!stack)
                dma_fence_put(&fence->base);
}

static void
xe_tlb_inval_fence_signal_unlocked(struct xe_tlb_inval_fence *fence)
{
        struct xe_tlb_inval *tlb_inval = fence->tlb_inval;

        spin_lock_irq(&tlb_inval->pending_lock);
        xe_tlb_inval_fence_signal(fence);
        spin_unlock_irq(&tlb_inval->pending_lock);
}

static void xe_tlb_inval_fence_timeout(struct work_struct *work)
{
        struct xe_tlb_inval *tlb_inval = container_of(work, struct xe_tlb_inval,
                                                      fence_tdr.work);
        struct xe_device *xe = tlb_inval->xe;
        struct xe_tlb_inval_fence *fence, *next;
        long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval);

        tlb_inval->ops->flush(tlb_inval);

        spin_lock_irq(&tlb_inval->pending_lock);
        list_for_each_entry_safe(fence, next,
                                 &tlb_inval->pending_fences, link) {
                s64 since_inval_ms = ktime_ms_delta(ktime_get(),
                                                    fence->inval_time);

                if (msecs_to_jiffies(since_inval_ms) < timeout_delay)
                        break;

                trace_xe_tlb_inval_fence_timeout(xe, fence);
                drm_err(&xe->drm,
                        "TLB invalidation fence timeout, seqno=%d recv=%d",
                        fence->seqno, tlb_inval->seqno_recv);

                fence->base.error = -ETIME;
                xe_tlb_inval_fence_signal(fence);
        }
        if (!list_empty(&tlb_inval->pending_fences))
                queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
                                   timeout_delay);
        spin_unlock_irq(&tlb_inval->pending_lock);
}

/**
 * tlb_inval_fini - Clean up TLB invalidation state
 * @drm: @drm_device
 * @arg: pointer to struct @xe_tlb_inval
 *
 * Cancel pending fence workers and clean up any additional
 * TLB invalidation state.
 */
static void tlb_inval_fini(struct drm_device *drm, void *arg)
{
        struct xe_tlb_inval *tlb_inval = arg;

        xe_tlb_inval_reset(tlb_inval);
}

static void primelockdep(struct xe_tlb_inval *tlb_inval)
{
        if (!IS_ENABLED(CONFIG_LOCKDEP))
                return;

        fs_reclaim_acquire(GFP_KERNEL);
        might_lock(&tlb_inval->seqno_lock);
        fs_reclaim_release(GFP_KERNEL);
}

/**
 * xe_gt_tlb_inval_init_early() - Initialize TLB invalidation state
 * @gt: GT structure
 *
 * Initialize TLB invalidation state, purely software initialization, should
 * be called once during driver load.
 *
 * Return: 0 on success, negative error code on error.
 */
int xe_gt_tlb_inval_init_early(struct xe_gt *gt)
{
        struct xe_device *xe = gt_to_xe(gt);
        struct xe_tlb_inval *tlb_inval = &gt->tlb_inval;
        int err;

        tlb_inval->xe = xe;
        tlb_inval->seqno = 1;
        INIT_LIST_HEAD(&tlb_inval->pending_fences);
        spin_lock_init(&tlb_inval->pending_lock);
        spin_lock_init(&tlb_inval->lock);
        INIT_DELAYED_WORK(&tlb_inval->fence_tdr, xe_tlb_inval_fence_timeout);

        err = drmm_mutex_init(&xe->drm, &tlb_inval->seqno_lock);
        if (err)
                return err;

        primelockdep(tlb_inval);

        tlb_inval->job_wq = drmm_alloc_ordered_workqueue(&xe->drm,
                                                         "gt-tbl-inval-job-wq",
                                                         WQ_MEM_RECLAIM);
        if (IS_ERR(tlb_inval->job_wq))
                return PTR_ERR(tlb_inval->job_wq);

        tlb_inval->timeout_wq = gt->ordered_wq;
        if (IS_ERR(tlb_inval->timeout_wq))
                return PTR_ERR(tlb_inval->timeout_wq);

        /* XXX: Blindly setting up backend to GuC */
        xe_guc_tlb_inval_init_early(&gt->uc.guc, tlb_inval);

        return drmm_add_action_or_reset(&xe->drm, tlb_inval_fini, tlb_inval);
}

/**
 * xe_tlb_inval_reset() - TLB invalidation reset
 * @tlb_inval: TLB invalidation client
 *
 * Signal any pending invalidation fences, should be called during a GT reset
 */
void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
{
        struct xe_tlb_inval_fence *fence, *next;
        int pending_seqno;

        /*
         * we can get here before the backends are even initialized if we're
         * wedging very early, in which case there are not going to be any
         * pendind fences so we can bail immediately.
         */
        if (!tlb_inval->ops->initialized(tlb_inval))
                return;

        /*
         * Backend is already disabled at this point. No new TLB requests can
         * appear.
         */

        mutex_lock(&tlb_inval->seqno_lock);
        spin_lock_irq(&tlb_inval->pending_lock);
        cancel_delayed_work(&tlb_inval->fence_tdr);
        /*
         * We might have various kworkers waiting for TLB flushes to complete
         * which are not tracked with an explicit TLB fence, however at this
         * stage that will never happen since the backend is already disabled,
         * so make sure we signal them here under the assumption that we have
         * completed a full GT reset.
         */
        if (tlb_inval->seqno == 1)
                pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1;
        else
                pending_seqno = tlb_inval->seqno - 1;
        WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno);

        list_for_each_entry_safe(fence, next,
                                 &tlb_inval->pending_fences, link)
                xe_tlb_inval_fence_signal(fence);
        spin_unlock_irq(&tlb_inval->pending_lock);
        mutex_unlock(&tlb_inval->seqno_lock);
}

/**
 * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout
 * @tlb_inval: TLB invalidation client
 *
 * Reset the TLB invalidation timeout timer.
 */
static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval)
{
        lockdep_assert_held(&tlb_inval->pending_lock);

        mod_delayed_work(system_wq, &tlb_inval->fence_tdr,
                         tlb_inval->ops->timeout_delay(tlb_inval));
}

static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
{
        int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);

        lockdep_assert_held(&tlb_inval->pending_lock);

        if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2))
                return false;

        if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2))
                return true;

        return seqno_recv >= seqno;
}

static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
{
        struct xe_tlb_inval *tlb_inval = fence->tlb_inval;

        fence->seqno = tlb_inval->seqno;
        trace_xe_tlb_inval_fence_send(tlb_inval->xe, fence);

        spin_lock_irq(&tlb_inval->pending_lock);
        fence->inval_time = ktime_get();
        list_add_tail(&fence->link, &tlb_inval->pending_fences);

        if (list_is_singular(&tlb_inval->pending_fences))
                queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
                                   tlb_inval->ops->timeout_delay(tlb_inval));
        spin_unlock_irq(&tlb_inval->pending_lock);

        tlb_inval->seqno = (tlb_inval->seqno + 1) %
                TLB_INVALIDATION_SEQNO_MAX;
        if (!tlb_inval->seqno)
                tlb_inval->seqno = 1;
}

#define xe_tlb_inval_issue(__tlb_inval, __fence, op, args...)   \
({                                                              \
        int __ret;                                              \
                                                                \
        xe_assert((__tlb_inval)->xe, (__tlb_inval)->ops);       \
        xe_assert((__tlb_inval)->xe, (__fence));                \
                                                                \
        mutex_lock(&(__tlb_inval)->seqno_lock);                 \
        xe_tlb_inval_fence_prep((__fence));                     \
        __ret = op((__tlb_inval), (__fence)->seqno, ##args);    \
        if (__ret < 0)                                          \
                xe_tlb_inval_fence_signal_unlocked((__fence));  \
        mutex_unlock(&(__tlb_inval)->seqno_lock);               \
                                                                \
        __ret == -ECANCELED ? 0 : __ret;                        \
})

/**
 * xe_tlb_inval_all() - Issue a TLB invalidation for all TLBs
 * @tlb_inval: TLB invalidation client
 * @fence: invalidation fence which will be signal on TLB invalidation
 * completion
 *
 * Issue a TLB invalidation for all TLBs. Completion of TLB is asynchronous and
 * caller can use the invalidation fence to wait for completion.
 *
 * Return: 0 on success, negative error code on error
 */
int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
                     struct xe_tlb_inval_fence *fence)
{
        return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->all);
}

/**
 * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT
 * @tlb_inval: TLB invalidation client
 *
 * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and
 * caller can use the invalidation fence to wait for completion.
 *
 * Return: 0 on success, negative error code on error
 */
int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
{
        struct xe_tlb_inval_fence fence, *fence_ptr = &fence;
        int ret;

        xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true);
        ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt);
        xe_tlb_inval_fence_wait(fence_ptr);

        return ret;
}

/**
 * xe_tlb_inval_range() - Issue a TLB invalidation for an address range
 * @tlb_inval: TLB invalidation client
 * @fence: invalidation fence which will be signal on TLB invalidation
 * completion
 * @start: start address
 * @end: end address
 * @asid: address space id
 * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush
 *
 * Issue a range based TLB invalidation if supported, if not fallback to a full
 * TLB invalidation. Completion of TLB is asynchronous and caller can use
 * the invalidation fence to wait for completion.
 *
 * Return: Negative error code on error, 0 on success
 */
int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
                       struct xe_tlb_inval_fence *fence, u64 start, u64 end,
                       u32 asid, struct drm_suballoc *prl_sa)
{
        return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
                                  start, end, asid, prl_sa);
}

/**
 * xe_tlb_inval_vm() - Issue a TLB invalidation for a VM
 * @tlb_inval: TLB invalidation client
 * @vm: VM to invalidate
 *
 * Invalidate entire VM's address space
 */
void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm)
{
        struct xe_tlb_inval_fence fence;
        u64 range = 1ull << vm->xe->info.va_bits;

        xe_tlb_inval_fence_init(tlb_inval, &fence, true);
        xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL);
        xe_tlb_inval_fence_wait(&fence);
}

/**
 * xe_tlb_inval_done_handler() - TLB invalidation done handler
 * @tlb_inval: TLB invalidation client
 * @seqno: seqno of invalidation that is done
 *
 * Update recv seqno, signal any TLB invalidation fences, and restart TDR
 */
void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
{
        struct xe_device *xe = tlb_inval->xe;
        struct xe_tlb_inval_fence *fence, *next;
        unsigned long flags;

        /*
         * This can also be run both directly from the IRQ handler and also in
         * process_g2h_msg(). Only one may process any individual CT message,
         * however the order they are processed here could result in skipping a
         * seqno. To handle that we just process all the seqnos from the last
         * seqno_recv up to and including the one in msg[0]. The delta should be
         * very small so there shouldn't be much of pending_fences we actually
         * need to iterate over here.
         *
         * From GuC POV we expect the seqnos to always appear in-order, so if we
         * see something later in the timeline we can be sure that anything
         * appearing earlier has already signalled, just that we have yet to
         * officially process the CT message like if racing against
         * process_g2h_msg().
         */
        spin_lock_irqsave(&tlb_inval->pending_lock, flags);
        if (seqno == TLB_INVALIDATION_SEQNO_INVALID) {
                xe_tlb_inval_reset_timeout(tlb_inval);
                spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
                return;
        }

        if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
                spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
                return;
        }

        WRITE_ONCE(tlb_inval->seqno_recv, seqno);

        list_for_each_entry_safe(fence, next,
                                 &tlb_inval->pending_fences, link) {
                trace_xe_tlb_inval_fence_recv(xe, fence);

                if (!xe_tlb_inval_seqno_past(tlb_inval, fence->seqno))
                        break;

                xe_tlb_inval_fence_signal(fence);
        }

        if (!list_empty(&tlb_inval->pending_fences))
                mod_delayed_work(tlb_inval->timeout_wq,
                                 &tlb_inval->fence_tdr,
                                 tlb_inval->ops->timeout_delay(tlb_inval));
        else
                cancel_delayed_work(&tlb_inval->fence_tdr);

        spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
}

static const char *
xe_inval_fence_get_driver_name(struct dma_fence *dma_fence)
{
        return "xe";
}

static const char *
xe_inval_fence_get_timeline_name(struct dma_fence *dma_fence)
{
        return "tlb_inval_fence";
}

static const struct dma_fence_ops inval_fence_ops = {
        .get_driver_name = xe_inval_fence_get_driver_name,
        .get_timeline_name = xe_inval_fence_get_timeline_name,
};

/**
 * xe_tlb_inval_fence_init() - Initialize TLB invalidation fence
 * @tlb_inval: TLB invalidation client
 * @fence: TLB invalidation fence to initialize
 * @stack: fence is stack variable
 *
 * Initialize TLB invalidation fence for use. xe_tlb_inval_fence_fini
 * will be automatically called when fence is signalled (all fences must signal),
 * even on error.
 */
void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
                             struct xe_tlb_inval_fence *fence,
                             bool stack)
{
        xe_pm_runtime_get_noresume(tlb_inval->xe);

        spin_lock_irq(&tlb_inval->lock);
        dma_fence_init(&fence->base, &inval_fence_ops, &tlb_inval->lock,
                       dma_fence_context_alloc(1), 1);
        spin_unlock_irq(&tlb_inval->lock);
        INIT_LIST_HEAD(&fence->link);
        if (stack)
                set_bit(FENCE_STACK_BIT, &fence->base.flags);
        else
                dma_fence_get(&fence->base);
        fence->tlb_inval = tlb_inval;
}

/**
 * xe_tlb_inval_idle() - Initialize TLB invalidation is idle
 * @tlb_inval: TLB invalidation client
 *
 * Check the TLB invalidation seqno to determine if it is idle (i.e., no TLB
 * invalidations are in flight). Expected to be called in the backend after the
 * fence has been added to the pending list, and takes this into account.
 *
 * Return: True if TLB invalidation client is idle, False otherwise
 */
bool xe_tlb_inval_idle(struct xe_tlb_inval *tlb_inval)
{
        lockdep_assert_held(&tlb_inval->seqno_lock);

        guard(spinlock_irq)(&tlb_inval->pending_lock);
        return list_is_singular(&tlb_inval->pending_fences);
}

/**
 * xe_tlb_inval_batch_wait() - Wait for all fences in a TLB invalidation batch
 * @batch: Batch of TLB invalidation fences to wait on
 *
 * Waits for every fence in @batch to signal, then resets @batch so it can be
 * reused for a subsequent invalidation.
 */
void xe_tlb_inval_batch_wait(struct xe_tlb_inval_batch *batch)
{
        struct xe_tlb_inval_fence *fence = &batch->fence[0];
        unsigned int i;

        for (i = 0; i < batch->num_fences; ++i)
                xe_tlb_inval_fence_wait(fence++);

        batch->num_fences = 0;
}

/**
 * xe_tlb_inval_range_tilemask_submit() - Submit TLB invalidations for an
 * address range on a tile mask
 * @xe: The xe device
 * @asid: Address space ID
 * @start: start address
 * @end: end address
 * @tile_mask: mask for which gt's issue tlb invalidation
 * @batch: Batch of tlb invalidate fences
 *
 * Issue a range based TLB invalidation for gt's in tilemask
 * If the function returns an error, there is no need to call
 * xe_tlb_inval_batch_wait() on @batch.
 *
 * Returns 0 for success, negative error code otherwise.
 */
int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid,
                                       u64 start, u64 end, u8 tile_mask,
                                       struct xe_tlb_inval_batch *batch)
{
        struct xe_tlb_inval_fence *fence = &batch->fence[0];
        struct xe_tile *tile;
        u32 fence_id = 0;
        u8 id;
        int err = 0;

        batch->num_fences = 0;
        if (!tile_mask)
                return 0;

        for_each_tile(tile, xe, id) {
                if (!(tile_mask & BIT(id)))
                        continue;

                xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
                                        &fence[fence_id], true);

                err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
                                         &fence[fence_id], start, end,
                                         asid, NULL);
                if (err)
                        goto wait;
                ++fence_id;

                if (!tile->media_gt)
                        continue;

                xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
                                        &fence[fence_id], true);

                err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
                                         &fence[fence_id], start, end,
                                         asid, NULL);
                if (err)
                        goto wait;
                ++fence_id;
        }

wait:
        batch->num_fences = fence_id;
        if (err)
                xe_tlb_inval_batch_wait(batch);

        return err;
}