// SPDX-License-Identifier: MIT
/*
 * Copyright © 2022 Intel Corporation
 */

#include "xe_guc_submit.h"

#include <linux/bitfield.h>
#include <linux/bitmap.h>
#include <linux/circ_buf.h>
#include <linux/delay.h>
#include <linux/dma-fence-array.h>
#include <linux/math64.h>

#include <drm/drm_managed.h>

#include "abi/guc_actions_abi.h"
#include "abi/guc_actions_slpc_abi.h"
#include "abi/guc_klvs_abi.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_devcoredump.h"
#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_force_wake.h"
#include "xe_gpu_scheduler.h"
#include "xe_gt.h"
#include "xe_gt_clock.h"
#include "xe_gt_printk.h"
#include "xe_guc.h"
#include "xe_guc_capture.h"
#include "xe_guc_ct.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_guc_id_mgr.h"
#include "xe_guc_klv_helpers.h"
#include "xe_guc_submit_types.h"
#include "xe_hw_engine.h"
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_map.h"
#include "xe_mocs.h"
#include "xe_pm.h"
#include "xe_ring_ops_types.h"
#include "xe_sched_job.h"
#include "xe_trace.h"
#include "xe_uc_fw.h"
#include "xe_vm.h"

#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN         6

static int guc_submit_reset_prepare(struct xe_guc *guc);

static struct xe_guc *
exec_queue_to_guc(struct xe_exec_queue *q)
{
        return &q->gt->uc.guc;
}

/*
 * Helpers for engine state, using an atomic as some of the bits can transition
 * as the same time (e.g. a suspend can be happning at the same time as schedule
 * engine done being processed).
 */
#define EXEC_QUEUE_STATE_REGISTERED             (1 << 0)
#define EXEC_QUEUE_STATE_ENABLED                (1 << 1)
#define EXEC_QUEUE_STATE_PENDING_ENABLE         (1 << 2)
#define EXEC_QUEUE_STATE_PENDING_DISABLE        (1 << 3)
#define EXEC_QUEUE_STATE_DESTROYED              (1 << 4)
#define EXEC_QUEUE_STATE_SUSPENDED              (1 << 5)
#define EXEC_QUEUE_STATE_RESET                  (1 << 6)
#define EXEC_QUEUE_STATE_KILLED                 (1 << 7)
#define EXEC_QUEUE_STATE_WEDGED                 (1 << 8)
#define EXEC_QUEUE_STATE_BANNED                 (1 << 9)
#define EXEC_QUEUE_STATE_PENDING_RESUME         (1 << 10)
#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND      (1 << 11)

static bool exec_queue_registered(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED;
}

static void set_exec_queue_registered(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
}

static void clear_exec_queue_registered(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
}

static bool exec_queue_enabled(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED;
}

static void set_exec_queue_enabled(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state);
}

static void clear_exec_queue_enabled(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state);
}

static bool exec_queue_pending_enable(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE;
}

static void set_exec_queue_pending_enable(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
}

static void clear_exec_queue_pending_enable(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
}

static bool exec_queue_pending_disable(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE;
}

static void set_exec_queue_pending_disable(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
}

static void clear_exec_queue_pending_disable(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
}

static bool exec_queue_destroyed(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED;
}

static void set_exec_queue_destroyed(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
}

static void clear_exec_queue_destroyed(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
}

static bool exec_queue_banned(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED;
}

static void set_exec_queue_banned(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
}

static bool exec_queue_suspended(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;
}

static void set_exec_queue_suspended(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);
}

static void clear_exec_queue_suspended(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);
}

static bool exec_queue_reset(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET;
}

static void set_exec_queue_reset(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state);
}

static bool exec_queue_killed(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED;
}

static void set_exec_queue_killed(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state);
}

static bool exec_queue_wedged(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED;
}

static void set_exec_queue_wedged(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state);
}

static bool exec_queue_pending_resume(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME;
}

static void set_exec_queue_pending_resume(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state);
}

static void clear_exec_queue_pending_resume(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state);
}

static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
{
        return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND;
}

static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
{
        atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
}

static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
{
        atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
}

static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
{
        return (atomic_read(&q->guc->state) &
                (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED |
                 EXEC_QUEUE_STATE_BANNED));
}

static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
{
        struct xe_guc *guc = arg;
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_gt *gt = guc_to_gt(guc);
        int ret;

        ret = wait_event_timeout(guc->submission_state.fini_wq,
                                 xa_empty(&guc->submission_state.exec_queue_lookup),
                                 HZ * 5);

        drain_workqueue(xe->destroy_wq);

        xe_gt_assert(gt, ret);

        xa_destroy(&guc->submission_state.exec_queue_lookup);
}

static void guc_submit_fini(void *arg)
{
        struct xe_guc *guc = arg;

        /* Forcefully kill any remaining exec queues */
        xe_guc_ct_stop(&guc->ct);
        guc_submit_reset_prepare(guc);
        xe_guc_softreset(guc);
        xe_guc_submit_stop(guc);
        xe_uc_fw_sanitize(&guc->fw);
        xe_guc_submit_pause_abort(guc);
}

static void guc_submit_wedged_fini(void *arg)
{
        struct xe_guc *guc = arg;
        struct xe_exec_queue *q;
        unsigned long index;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                if (exec_queue_wedged(q)) {
                        mutex_unlock(&guc->submission_state.lock);
                        xe_exec_queue_put(q);
                        mutex_lock(&guc->submission_state.lock);
                }
        }
        mutex_unlock(&guc->submission_state.lock);
}

static const struct xe_exec_queue_ops guc_exec_queue_ops;

static void primelockdep(struct xe_guc *guc)
{
        if (!IS_ENABLED(CONFIG_LOCKDEP))
                return;

        fs_reclaim_acquire(GFP_KERNEL);

        mutex_lock(&guc->submission_state.lock);
        mutex_unlock(&guc->submission_state.lock);

        fs_reclaim_release(GFP_KERNEL);
}

/**
 * xe_guc_submit_init() - Initialize GuC submission.
 * @guc: the &xe_guc to initialize
 * @num_ids: number of GuC context IDs to use
 *
 * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all
 * GuC context IDs supported by the GuC firmware should be used for submission.
 *
 * Only VF drivers will have to provide explicit number of GuC context IDs
 * that they can use for submission.
 *
 * Return: 0 on success or a negative error code on failure.
 */
int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids)
{
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_gt *gt = guc_to_gt(guc);
        int err;

        err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock);
        if (err)
                return err;

        err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids);
        if (err)
                return err;

        gt->exec_queue_ops = &guc_exec_queue_ops;

        xa_init(&guc->submission_state.exec_queue_lookup);

        init_waitqueue_head(&guc->submission_state.fini_wq);

        primelockdep(guc);

        guc->submission_state.initialized = true;

        err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc);
        if (err)
                return err;

        return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc);
}

/*
 * Given that we want to guarantee enough RCS throughput to avoid missing
 * frames, we set the yield policy to 20% of each 80ms interval.
 */
#define RC_YIELD_DURATION       80      /* in ms */
#define RC_YIELD_RATIO          20      /* in percent */
static u32 *emit_render_compute_yield_klv(u32 *emit)
{
        *emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD);
        *emit++ = RC_YIELD_DURATION;
        *emit++ = RC_YIELD_RATIO;

        return emit;
}

#define SCHEDULING_POLICY_MAX_DWORDS 16
static int guc_init_global_schedule_policy(struct xe_guc *guc)
{
        u32 data[SCHEDULING_POLICY_MAX_DWORDS];
        u32 *emit = data;
        u32 count = 0;
        int ret;

        if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
                return 0;

        *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;

        if (CCS_INSTANCES(guc_to_gt(guc)))
                emit = emit_render_compute_yield_klv(emit);

        count = emit - data;
        if (count > 1) {
                xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS);

                ret = xe_guc_ct_send_block(&guc->ct, data, count);
                if (ret < 0) {
                        xe_gt_err(guc_to_gt(guc),
                                  "failed to enable GuC scheduling policies: %pe\n",
                                  ERR_PTR(ret));
                        return ret;
                }
        }

        return 0;
}

int xe_guc_submit_enable(struct xe_guc *guc)
{
        int ret;

        ret = guc_init_global_schedule_policy(guc);
        if (ret)
                return ret;

        guc->submission_state.enabled = true;

        return 0;
}

void xe_guc_submit_disable(struct xe_guc *guc)
{
        guc->submission_state.enabled = false;
}

static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count)
{
        int i;

        lockdep_assert_held(&guc->submission_state.lock);

        for (i = 0; i < xa_count; ++i)
                xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i);

        xe_guc_id_mgr_release_locked(&guc->submission_state.idm,
                                     q->guc->id, q->width);

        if (xa_empty(&guc->submission_state.exec_queue_lookup))
                wake_up(&guc->submission_state.fini_wq);
}

static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
{
        int ret;
        int i;

        /*
         * Must use GFP_NOWAIT as this lock is in the dma fence signalling path,
         * worse case user gets -ENOMEM on engine create and has to try again.
         *
         * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent
         * failure.
         */
        lockdep_assert_held(&guc->submission_state.lock);

        ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm,
                                           q->width);
        if (ret < 0)
                return ret;

        q->guc->id = ret;

        for (i = 0; i < q->width; ++i) {
                ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup,
                                      q->guc->id + i, q, GFP_NOWAIT));
                if (ret)
                        goto err_release;
        }

        return 0;

err_release:
        __release_guc_id(guc, q, i);

        return ret;
}

static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
{
        mutex_lock(&guc->submission_state.lock);
        __release_guc_id(guc, q, q->width);
        mutex_unlock(&guc->submission_state.lock);
}

struct exec_queue_policy {
        u32 count;
        struct guc_update_exec_queue_policy h2g;
};

static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy)
{
        size_t bytes = sizeof(policy->h2g.header) +
                       (sizeof(policy->h2g.klv[0]) * policy->count);

        return bytes / sizeof(u32);
}

static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy,
                                              u16 guc_id)
{
        policy->h2g.header.action =
                XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
        policy->h2g.header.guc_id = guc_id;
        policy->count = 0;
}

#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \
static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \
                                           u32 data) \
{ \
        XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
\
        policy->h2g.klv[policy->count].kl = \
                FIELD_PREP(GUC_KLV_0_KEY, \
                           GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
                FIELD_PREP(GUC_KLV_0_LEN, 1); \
        policy->h2g.klv[policy->count].value = data; \
        policy->count++; \
}

MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY)
MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY)
#undef MAKE_EXEC_QUEUE_POLICY_ADD

static const int xe_exec_queue_prio_to_guc[] = {
        [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL,
        [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL,
        [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH,
        [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH,
};

static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)
{
        struct exec_queue_policy policy;
        enum xe_exec_queue_priority prio = q->sched_props.priority;
        u32 timeslice_us = q->sched_props.timeslice_us;
        u32 slpc_exec_queue_freq_req = 0;
        u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;

        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) &&
                     !xe_exec_queue_is_multi_queue_secondary(q));

        if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)
                slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;

        __guc_exec_queue_policy_start_klv(&policy, q->guc->id);
        __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]);
        __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us);
        __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us);
        __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy,
                                                             slpc_exec_queue_freq_req);

        xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
                       __guc_exec_queue_policy_action_size(&policy), 0, 0);
}

static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q)
{
        struct exec_queue_policy policy;

        xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q));

        __guc_exec_queue_policy_start_klv(&policy, q->guc->id);
        __guc_exec_queue_policy_add_preemption_timeout(&policy, 1);

        xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
                       __guc_exec_queue_policy_action_size(&policy), 0, 0);
}

static bool vf_recovery(struct xe_guc *guc)
{
        return xe_gt_recovery_pending(guc_to_gt(guc));
}

static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);

        /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
        wake_up_all(&xe->ufence_wq);

        xe_sched_tdr_queue_imm(&q->guc->sched);
}

static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
{
        struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
        struct xe_exec_queue_group *group = q->multi_queue.group;
        struct xe_exec_queue *eq;

        xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
                     xe_exec_queue_is_multi_queue(q));

        /* Group banned, skip timeout check in TDR */
        WRITE_ONCE(group->banned, true);
        xe_guc_exec_queue_trigger_cleanup(primary);

        mutex_lock(&group->list_lock);
        list_for_each_entry(eq, &group->list, multi_queue.link)
                xe_guc_exec_queue_trigger_cleanup(eq);
        mutex_unlock(&group->list_lock);
}

static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q)
{
        if (xe_exec_queue_is_multi_queue(q)) {
                struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
                struct xe_exec_queue_group *group = q->multi_queue.group;
                struct xe_exec_queue *eq;

                /* Group banned, skip timeout check in TDR */
                WRITE_ONCE(group->banned, true);

                set_exec_queue_reset(primary);
                if (!exec_queue_banned(primary))
                        xe_guc_exec_queue_trigger_cleanup(primary);

                mutex_lock(&group->list_lock);
                list_for_each_entry(eq, &group->list, multi_queue.link) {
                        set_exec_queue_reset(eq);
                        if (!exec_queue_banned(eq))
                                xe_guc_exec_queue_trigger_cleanup(eq);
                }
                mutex_unlock(&group->list_lock);
        } else {
                set_exec_queue_reset(q);
                if (!exec_queue_banned(q))
                        xe_guc_exec_queue_trigger_cleanup(q);
        }
}

static void set_exec_queue_group_banned(struct xe_exec_queue *q)
{
        struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
        struct xe_exec_queue_group *group = q->multi_queue.group;
        struct xe_exec_queue *eq;

        /* Ban all queues of the multi-queue group */
        xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
                     xe_exec_queue_is_multi_queue(q));
        set_exec_queue_banned(primary);

        mutex_lock(&group->list_lock);
        list_for_each_entry(eq, &group->list, multi_queue.link)
                set_exec_queue_banned(eq);
        mutex_unlock(&group->list_lock);
}

/* Helper for context registration H2G */
struct guc_ctxt_registration_info {
        u32 flags;
        u32 context_idx;
        u32 engine_class;
        u32 engine_submit_mask;
        u32 wq_desc_lo;
        u32 wq_desc_hi;
        u32 wq_base_lo;
        u32 wq_base_hi;
        u32 wq_size;
        u32 cgp_lo;
        u32 cgp_hi;
        u32 hwlrca_lo;
        u32 hwlrca_hi;
};

#define parallel_read(xe_, map_, field_) \
        xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
                        field_)
#define parallel_write(xe_, map_, field_, val_) \
        xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
                        field_, val_)

/**
 * DOC: Multi Queue Group GuC interface
 *
 * The multi queue group coordination between KMD and GuC is through a software
 * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page
 * allocated in the global GTT.
 *
 * CGP format:
 *
 * +-----------+---------------------------+---------------------------------------------+
 * | DWORD     | Name                      | Description                                 |
 * +-----------+---------------------------+---------------------------------------------+
 * | 0         | Version                   | Bits [15:8]=Major ver, [7:0]=Minor ver      |
 * +-----------+---------------------------+---------------------------------------------+
 * | 1..15     | RESERVED                  | MBZ                                         |
 * +-----------+---------------------------+---------------------------------------------+
 * | 16        | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0             |
 * +-----------+---------------------------+---------------------------------------------+
 * | 17        | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32            |
 * +-----------+---------------------------+---------------------------------------------+
 * | 18..31    | RESERVED                  | MBZ                                         |
 * +-----------+---------------------------+---------------------------------------------+
 * | 32        | Q0CD_DW0                  | Queue 0 context LRC descriptor lower DWORD  |
 * +-----------+---------------------------+---------------------------------------------+
 * | 33        | Q0ContextIndex            | Context ID for Queue 0                      |
 * +-----------+---------------------------+---------------------------------------------+
 * | 34        | Q1CD_DW0                  | Queue 1 context LRC descriptor lower DWORD  |
 * +-----------+---------------------------+---------------------------------------------+
 * | 35        | Q1ContextIndex            | Context ID for Queue 1                      |
 * +-----------+---------------------------+---------------------------------------------+
 * | ...       |...                        | ...                                         |
 * +-----------+---------------------------+---------------------------------------------+
 * | 158       | Q63CD_DW0                 | Queue 63 context LRC descriptor lower DWORD |
 * +-----------+---------------------------+---------------------------------------------+
 * | 159       | Q63ContextIndex           | Context ID for Queue 63                     |
 * +-----------+---------------------------+---------------------------------------------+
 * | 160..1024 | RESERVED                  | MBZ                                         |
 * +-----------+---------------------------+---------------------------------------------+
 *
 * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified
 * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies
 * the CGP address. When the secondary queues are added to the group, the CGP is
 * updated with entry for that queue and GuC is notified through the H2G interface
 * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages
 * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also
 * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any
 * error in the CGP. Only one of these CGP update messages can be outstanding
 * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW*
 * fields indicate which queue entry is being updated in the CGP.
 *
 * The primary queue (Q0) represents the multi queue group context in GuC and
 * submission on any queue of the group must be through Q0 GuC interface only.
 *
 * As it is not required to register secondary queues with GuC, the secondary queue
 * context ids in the CGP are populated with Q0 context id.
 */

#define CGP_VERSION_MAJOR_SHIFT 8

static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
                                               struct xe_exec_queue *q)
{
        struct xe_exec_queue_group *group = q->multi_queue.group;
        u32 guc_id = group->primary->guc->id;

        /* Currently implementing CGP version 1.0 */
        xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32,
                  1 << CGP_VERSION_MAJOR_SHIFT);

        xe_map_wr(xe, &group->cgp_bo->vmap,
                  (32 + q->multi_queue.pos * 2) * sizeof(u32),
                  u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0])));

        xe_map_wr(xe, &group->cgp_bo->vmap,
                  (33 + q->multi_queue.pos * 2) * sizeof(u32),
                  u32, guc_id);

        if (q->multi_queue.pos / 32) {
                xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32),
                          u32, BIT(q->multi_queue.pos % 32));
                xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0);
        } else {
                xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32),
                          u32, BIT(q->multi_queue.pos));
                xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0);
        }
}

static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
                                             struct xe_exec_queue *q,
                                             const u32 *action, u32 len)
{
        struct xe_exec_queue_group *group = q->multi_queue.group;
        struct xe_device *xe = guc_to_xe(guc);
        long ret;

        /*
         * As all queues of a multi queue group use single drm scheduler
         * submit workqueue, CGP synchronization with GuC are serialized.
         * Hence, no locking is required here.
         * Wait for any pending CGP_SYNC_DONE response before updating the
         * CGP page and sending CGP_SYNC message.
         *
         * FIXME: Support VF migration
         */
        ret = wait_event_timeout(guc->ct.wq,
                                 !READ_ONCE(group->sync_pending) ||
                                 xe_guc_read_stopped(guc), HZ);
        if (!ret || xe_guc_read_stopped(guc)) {
                /* CGP_SYNC failed. Reset gt, cleanup the group */
                xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
                set_exec_queue_group_banned(q);
                xe_gt_reset_async(q->gt);
                xe_guc_exec_queue_group_trigger_cleanup(q);
                return;
        }

        xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority);
        xe_guc_exec_queue_group_cgp_update(xe, q);

        WRITE_ONCE(group->sync_pending, true);
        xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
}

static void __register_exec_queue_group(struct xe_guc *guc,
                                        struct xe_exec_queue *q,
                                        struct guc_ctxt_registration_info *info)
{
#define MAX_MULTI_QUEUE_REG_SIZE        (8)
        u32 action[MAX_MULTI_QUEUE_REG_SIZE];
        int len = 0;

        action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE;
        action[len++] = info->flags;
        action[len++] = info->context_idx;
        action[len++] = info->engine_class;
        action[len++] = info->engine_submit_mask;
        action[len++] = 0; /* Reserved */
        action[len++] = info->cgp_lo;
        action[len++] = info->cgp_hi;

        xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE);
#undef MAX_MULTI_QUEUE_REG_SIZE

        /*
         * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a
         * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
         * from guc.
         */
        xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
}

static void xe_guc_exec_queue_group_add(struct xe_guc *guc,
                                        struct xe_exec_queue *q)
{
#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE  (2)
        u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
        int len = 0;

        xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q));

        action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
        action[len++] = q->multi_queue.group->primary->guc->id;

        xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE

        /*
         * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a
         * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
         * from guc.
         */
        xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
}

static void __register_mlrc_exec_queue(struct xe_guc *guc,
                                       struct xe_exec_queue *q,
                                       struct guc_ctxt_registration_info *info)
{
#define MAX_MLRC_REG_SIZE      (13 + XE_HW_ENGINE_MAX_INSTANCE * 2)
        u32 action[MAX_MLRC_REG_SIZE];
        int len = 0;
        int i;

        xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q));

        action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
        action[len++] = info->flags;
        action[len++] = info->context_idx;
        action[len++] = info->engine_class;
        action[len++] = info->engine_submit_mask;
        action[len++] = info->wq_desc_lo;
        action[len++] = info->wq_desc_hi;
        action[len++] = info->wq_base_lo;
        action[len++] = info->wq_base_hi;
        action[len++] = info->wq_size;
        action[len++] = q->width;
        action[len++] = info->hwlrca_lo;
        action[len++] = info->hwlrca_hi;

        for (i = 1; i < q->width; ++i) {
                struct xe_lrc *lrc = q->lrc[i];

                action[len++] = lower_32_bits(xe_lrc_descriptor(lrc));
                action[len++] = upper_32_bits(xe_lrc_descriptor(lrc));
        }

        /* explicitly checks some fields that we might fixup later */
        xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]);
        xe_gt_assert(guc_to_gt(guc), info->wq_base_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]);
        xe_gt_assert(guc_to_gt(guc), q->width ==
                     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]);
        xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]);
        xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE);
#undef MAX_MLRC_REG_SIZE

        xe_guc_ct_send(&guc->ct, action, len, 0, 0);
}

static void __register_exec_queue(struct xe_guc *guc,
                                  struct guc_ctxt_registration_info *info)
{
        u32 action[] = {
                XE_GUC_ACTION_REGISTER_CONTEXT,
                info->flags,
                info->context_idx,
                info->engine_class,
                info->engine_submit_mask,
                info->wq_desc_lo,
                info->wq_desc_hi,
                info->wq_base_lo,
                info->wq_base_hi,
                info->wq_size,
                info->hwlrca_lo,
                info->hwlrca_hi,
        };

        /* explicitly checks some fields that we might fixup later */
        xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]);
        xe_gt_assert(guc_to_gt(guc), info->wq_base_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]);
        xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo ==
                     action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]);

        xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
}

static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_lrc *lrc = q->lrc[0];
        struct guc_ctxt_registration_info info;

        xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q));
        xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT);

        memset(&info, 0, sizeof(info));
        info.context_idx = q->guc->id;
        info.engine_class = xe_engine_class_to_guc_class(q->class);
        info.engine_submit_mask = q->logical_mask;
        info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc));
        info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc));
        info.flags = CONTEXT_REGISTRATION_FLAG_KMD |
                FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);

        if (xe_exec_queue_is_multi_queue(q)) {
                struct xe_exec_queue_group *group = q->multi_queue.group;

                info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo);
                info.cgp_hi = 0;
        }

        if (xe_exec_queue_is_parallel(q)) {
                u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
                struct iosys_map map = xe_lrc_parallel_map(lrc);

                info.wq_desc_lo = lower_32_bits(ggtt_addr +
                        offsetof(struct guc_submit_parallel_scratch, wq_desc));
                info.wq_desc_hi = upper_32_bits(ggtt_addr +
                        offsetof(struct guc_submit_parallel_scratch, wq_desc));
                info.wq_base_lo = lower_32_bits(ggtt_addr +
                        offsetof(struct guc_submit_parallel_scratch, wq[0]));
                info.wq_base_hi = upper_32_bits(ggtt_addr +
                        offsetof(struct guc_submit_parallel_scratch, wq[0]));
                info.wq_size = WQ_SIZE;

                q->guc->wqi_head = 0;
                q->guc->wqi_tail = 0;
                xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE);
                parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE);
        }

        set_exec_queue_registered(q);
        trace_xe_exec_queue_register(q);
        if (xe_exec_queue_is_multi_queue_primary(q))
                __register_exec_queue_group(guc, q, &info);
        else if (xe_exec_queue_is_parallel(q))
                __register_mlrc_exec_queue(guc, q, &info);
        else if (!xe_exec_queue_is_multi_queue_secondary(q))
                __register_exec_queue(guc, &info);

        if (!xe_exec_queue_is_multi_queue_secondary(q))
                init_policies(guc, q);

        if (xe_exec_queue_is_multi_queue_secondary(q))
                xe_guc_exec_queue_group_add(guc, q);
}

static u32 wq_space_until_wrap(struct xe_exec_queue *q)
{
        return (WQ_SIZE - q->guc->wqi_tail);
}

static inline void relaxed_ms_sleep(unsigned int delay_ms)
{
        unsigned long min_us, max_us;

        if (!delay_ms)
                return;

        if (delay_ms > 20) {
                msleep(delay_ms);
                return;
        }

        min_us = mul_u32_u32(delay_ms, 1000);
        max_us = min_us + 500;

        usleep_range(min_us, max_us);
}

static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
        unsigned int sleep_period_ms = 1, sleep_total_ms = 0;

#define AVAILABLE_SPACE \
        CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)
        if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
try_again:
                q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);
                if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
                        if (sleep_total_ms > 2000) {
                                xe_gt_reset_async(q->gt);
                                return -ENODEV;
                        }

                        msleep(sleep_period_ms);
                        sleep_total_ms += sleep_period_ms;
                        if (sleep_period_ms < 64)
                                sleep_period_ms <<= 1;
                        goto try_again;
                }
        }
#undef AVAILABLE_SPACE

        return 0;
}

static int wq_noop_append(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
        u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1;

        if (wq_wait_for_space(q, wq_space_until_wrap(q)))
                return -ENODEV;

        xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw));

        parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)],
                       FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
                       FIELD_PREP(WQ_LEN_MASK, len_dw));
        q->guc->wqi_tail = 0;

        return 0;
}

static void wq_item_append(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
#define WQ_HEADER_SIZE  4       /* Includes 1 LRC address too */
        u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)];
        u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32);
        u32 len_dw = (wqi_size / sizeof(u32)) - 1;
        int i = 0, j;

        if (wqi_size > wq_space_until_wrap(q)) {
                if (wq_noop_append(q))
                        return;
        }
        if (wq_wait_for_space(q, wqi_size))
                return;

        wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
                FIELD_PREP(WQ_LEN_MASK, len_dw);
        wqi[i++] = xe_lrc_descriptor(q->lrc[0]);
        wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) |
                FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64));
        wqi[i++] = 0;
        for (j = 1; j < q->width; ++j) {
                struct xe_lrc *lrc = q->lrc[j];

                wqi[i++] = lrc->ring.tail / sizeof(u64);
        }

        xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32));

        iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch,
                                      wq[q->guc->wqi_tail / sizeof(u32)]));
        xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size);
        q->guc->wqi_tail += wqi_size;
        xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE);

        xe_device_wmb(xe);

        map = xe_lrc_parallel_map(q->lrc[0]);
        parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail);
}

#define RESUME_PENDING  ~0x0ull
static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_lrc *lrc = q->lrc[0];
        u32 action[3];
        u32 g2h_len = 0;
        u32 num_g2h = 0;
        int len = 0;
        bool extra_submit = false;

        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));

        if (!job->restore_replay || job->last_replay) {
                if (xe_exec_queue_is_parallel(q))
                        wq_item_append(q);
                else if (!exec_queue_idle_skip_suspend(q))
                        xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
                job->last_replay = false;
        }

        if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
                return;

        /*
         * All queues in a multi-queue group will use the primary queue
         * of the group to interface with GuC.
         */
        q = xe_exec_queue_multi_queue_primary(q);

        if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {
                action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
                action[len++] = q->guc->id;
                action[len++] = GUC_CONTEXT_ENABLE;
                g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
                num_g2h = 1;
                if (xe_exec_queue_is_parallel(q))
                        extra_submit = true;

                q->guc->resume_time = RESUME_PENDING;
                set_exec_queue_pending_enable(q);
                set_exec_queue_enabled(q);
                trace_xe_exec_queue_scheduling_enable(q);
        } else {
                action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
                action[len++] = q->guc->id;
                trace_xe_exec_queue_submit(q);
        }

        xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);

        if (extra_submit) {
                len = 0;
                action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
                action[len++] = q->guc->id;
                trace_xe_exec_queue_submit(q);

                xe_guc_ct_send(&guc->ct, action, len, 0, 0);
        }
}

static struct dma_fence *
guc_exec_queue_run_job(struct drm_sched_job *drm_job)
{
        struct xe_sched_job *job = to_xe_sched_job(drm_job);
        struct xe_exec_queue *q = job->q;
        struct xe_guc *guc = exec_queue_to_guc(q);
        bool killed_or_banned_or_wedged =
                exec_queue_killed_or_banned_or_wedged(q);

        xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||
                     exec_queue_banned(q) || exec_queue_suspended(q));

        trace_xe_sched_job_run(job);

        if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
                if (xe_exec_queue_is_multi_queue_secondary(q)) {
                        struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);

                        if (exec_queue_killed_or_banned_or_wedged(primary)) {
                                killed_or_banned_or_wedged = true;
                                goto run_job_out;
                        }

                        if (!exec_queue_registered(primary))
                                register_exec_queue(primary, GUC_CONTEXT_NORMAL);
                }

                if (!exec_queue_registered(q))
                        register_exec_queue(q, GUC_CONTEXT_NORMAL);
                if (!job->restore_replay)
                        q->ring_ops->emit_job(job);
                submit_exec_queue(q, job);
                job->restore_replay = false;
        }

run_job_out:

        return job->fence;
}

static void guc_exec_queue_free_job(struct drm_sched_job *drm_job)
{
        struct xe_sched_job *job = to_xe_sched_job(drm_job);

        trace_xe_sched_job_free(job);
        xe_sched_job_put(job);
}

int xe_guc_read_stopped(struct xe_guc *guc)
{
        return atomic_read(&guc->submission_state.stopped);
}

static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
                                                    struct xe_exec_queue *q,
                                                    u32 runnable_state);
static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q);

#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable)                    \
        u32 action[] = {                                                \
                XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET,                   \
                q->guc->id,                                             \
                GUC_CONTEXT_##enable_disable,                           \
        }

static void disable_scheduling_deregister(struct xe_guc *guc,
                                          struct xe_exec_queue *q)
{
        MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
        int ret;

        if (!xe_exec_queue_is_multi_queue_secondary(q))
                set_min_preemption_timeout(guc, q);

        smp_rmb();
        ret = wait_event_timeout(guc->ct.wq,
                                 (!exec_queue_pending_enable(q) &&
                                  !exec_queue_pending_disable(q)) ||
                                         xe_guc_read_stopped(guc) ||
                                         vf_recovery(guc),
                                 HZ * 5);
        if (!ret && !vf_recovery(guc)) {
                struct xe_gpu_scheduler *sched = &q->guc->sched;

                xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n");
                xe_sched_submission_start(sched);
                xe_gt_reset_async(q->gt);
                xe_sched_tdr_queue_imm(sched);
                return;
        }

        clear_exec_queue_enabled(q);
        set_exec_queue_pending_disable(q);
        set_exec_queue_destroyed(q);
        trace_xe_exec_queue_scheduling_disable(q);

        /*
         * Reserve space for both G2H here as the 2nd G2H is sent from a G2H
         * handler and we are not allowed to reserved G2H space in handlers.
         */
        if (xe_exec_queue_is_multi_queue_secondary(q))
                handle_multi_queue_secondary_sched_done(guc, q, 0);
        else
                xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
                               G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
                               G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
}

/**
 * xe_guc_submit_wedge() - Wedge GuC submission
 * @guc: the GuC object
 *
 * Save exec queue's registered with GuC state by taking a ref to each queue.
 * Register a DRMM handler to drop refs upon driver unload.
 */
void xe_guc_submit_wedge(struct xe_guc *guc)
{
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_gt *gt = guc_to_gt(guc);
        struct xe_exec_queue *q;
        unsigned long index;
        int err;

        xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);

        /*
         * If device is being wedged even before submission_state is
         * initialized, there's nothing to do here.
         */
        if (!guc->submission_state.initialized)
                return;

        if (xe->wedged.mode == 2) {
                err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
                                               guc_submit_wedged_fini, guc);
                if (err) {
                        xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; "
                                  "Although device is wedged.\n");
                        return;
                }

                mutex_lock(&guc->submission_state.lock);
                xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
                        if (xe_exec_queue_get_unless_zero(q))
                                set_exec_queue_wedged(q);
                mutex_unlock(&guc->submission_state.lock);
        } else {
                /* Forcefully kill any remaining exec queues, signal fences */
                guc_submit_reset_prepare(guc);
                xe_guc_submit_stop(guc);
                xe_guc_softreset(guc);
                xe_uc_fw_sanitize(&guc->fw);
                xe_guc_submit_pause_abort(guc);
        }
}

static bool guc_submit_hint_wedged(struct xe_guc *guc)
{
        struct xe_device *xe = guc_to_xe(guc);

        if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)
                return false;

        if (xe_device_wedged(xe))
                return true;

        xe_device_declare_wedged(xe);

        return true;
}

#define ADJUST_FIVE_PERCENT(__t)        mul_u64_u32_div(__t, 105, 100)

static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job)
{
        struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q));
        u32 ctx_timestamp, ctx_job_timestamp;
        u32 timeout_ms = q->sched_props.job_timeout_ms;
        u32 diff;
        u64 running_time_ms;

        if (!xe_sched_job_started(job)) {
                xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started",
                           xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
                           q->guc->id);

                return xe_sched_invalidate_job(job, 2);
        }

        ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0]));
        if (ctx_timestamp == job->sample_timestamp) {
                if (IS_SRIOV_VF(gt_to_xe(gt)))
                        xe_gt_notice(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
                                     xe_sched_job_seqno(job),
                                     xe_sched_job_lrc_seqno(job), q->guc->id);
                else
                        xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
                                   xe_sched_job_seqno(job),
                                   xe_sched_job_lrc_seqno(job), q->guc->id);

                return xe_sched_invalidate_job(job, 0);
        }

        job->sample_timestamp = ctx_timestamp;
        ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]);

        /*
         * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch
         * possible overflows with a high timeout.
         */
        xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC);

        diff = ctx_timestamp - ctx_job_timestamp;

        /*
         * Ensure timeout is within 5% to account for an GuC scheduling latency
         */
        running_time_ms =
                ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff));

        xe_gt_dbg(gt,
                  "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x",
                  xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
                  q->guc->id, running_time_ms, timeout_ms, diff);

        return running_time_ms >= timeout_ms;
}

static void enable_scheduling(struct xe_exec_queue *q)
{
        MAKE_SCHED_CONTEXT_ACTION(q, ENABLE);
        struct xe_guc *guc = exec_queue_to_guc(q);
        int ret;

        xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));

        set_exec_queue_pending_enable(q);
        set_exec_queue_enabled(q);
        trace_xe_exec_queue_scheduling_enable(q);

        if (xe_exec_queue_is_multi_queue_secondary(q))
                handle_multi_queue_secondary_sched_done(guc, q, 1);
        else
                xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
                               G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);

        ret = wait_event_timeout(guc->ct.wq,
                                 !exec_queue_pending_enable(q) ||
                                 xe_guc_read_stopped(guc) ||
                                 vf_recovery(guc), HZ * 5);
        if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) {
                xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond");
                set_exec_queue_banned(q);
                xe_gt_reset_async(q->gt);
                xe_sched_tdr_queue_imm(&q->guc->sched);
        }
}

static void disable_scheduling(struct xe_exec_queue *q, bool immediate)
{
        MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
        struct xe_guc *guc = exec_queue_to_guc(q);

        xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));

        if (immediate && !xe_exec_queue_is_multi_queue_secondary(q))
                set_min_preemption_timeout(guc, q);
        clear_exec_queue_enabled(q);
        set_exec_queue_pending_disable(q);
        trace_xe_exec_queue_scheduling_disable(q);

        if (xe_exec_queue_is_multi_queue_secondary(q))
                handle_multi_queue_secondary_sched_done(guc, q, 0);
        else
                xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
                               G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
}

static enum drm_gpu_sched_stat
guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
{
        struct xe_sched_job *job = to_xe_sched_job(drm_job);
        struct drm_sched_job *tmp_job;
        struct xe_exec_queue *q = job->q;
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_guc *guc = exec_queue_to_guc(q);
        const char *process_name = "no process";
        struct xe_device *xe = guc_to_xe(guc);
        int err = -ETIME;
        pid_t pid = -1;
        bool wedged = false, skip_timeout_check;

        xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));

        /*
         * TDR has fired before free job worker. Common if exec queue
         * immediately closed after last fence signaled. Add back to pending
         * list so job can be freed and kick scheduler ensuring free job is not
         * lost.
         */
        if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) ||
            vf_recovery(guc))
                return DRM_GPU_SCHED_STAT_NO_HANG;

        /* Kill the run_job entry point */
        xe_sched_submission_stop(sched);

        /* Must check all state after stopping scheduler */
        skip_timeout_check = exec_queue_reset(q) ||
                exec_queue_killed_or_banned_or_wedged(q);

        /* Skip timeout check if multi-queue group is banned */
        if (xe_exec_queue_is_multi_queue(q) &&
            READ_ONCE(q->multi_queue.group->banned))
                skip_timeout_check = true;

        /* LR jobs can only get here if queue has been killed or hit an error */
        if (xe_exec_queue_is_lr(q))
                xe_gt_assert(guc_to_gt(guc), skip_timeout_check);

        /*
         * FIXME: In multi-queue scenario, the TDR must ensure that the whole
         * multi-queue group is off the HW before signaling the fences to avoid
         * possible memory corruptions. This means disabling scheduling on the
         * primary queue before or during the secondary queue's TDR. Need to
         * implement this in least obtrusive way.
         */

        /*
         * If devcoredump not captured and GuC capture for the job is not ready
         * do manual capture first and decide later if we need to use it
         */
        if (!exec_queue_killed(q) && !xe->devcoredump.captured &&
            !xe_guc_capture_get_matching_and_lock(q)) {
                /* take force wake before engine register manual capture */
                CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
                if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
                        xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");

                xe_engine_snapshot_capture_for_queue(q);
        }

        /*
         * Check if job is actually timed out, if so restart job execution and TDR
         */
        if (!skip_timeout_check && !check_timeout(q, job))
                goto rearm;

        if (!exec_queue_killed(q))
                wedged = guc_submit_hint_wedged(exec_queue_to_guc(q));

        set_exec_queue_banned(q);

        /* Kick job / queue off hardware */
        if (!wedged && (exec_queue_enabled(q) || exec_queue_pending_disable(q))) {
                int ret;

                if (exec_queue_reset(q))
                        err = -EIO;

                if (xe_uc_fw_is_running(&guc->fw)) {
                        /*
                         * Wait for any pending G2H to flush out before
                         * modifying state
                         */
                        ret = wait_event_timeout(guc->ct.wq,
                                                 (!exec_queue_pending_enable(q) &&
                                                  !exec_queue_pending_disable(q)) ||
                                                 xe_guc_read_stopped(guc) ||
                                                 vf_recovery(guc), HZ * 5);
                        if (vf_recovery(guc))
                                goto handle_vf_resume;
                        if (!ret || xe_guc_read_stopped(guc))
                                goto trigger_reset;

                        disable_scheduling(q, skip_timeout_check);
                }

                /*
                 * Must wait for scheduling to be disabled before signalling
                 * any fences, if GT broken the GT reset code should signal us.
                 *
                 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault
                 * error) messages which can cause the schedule disable to get
                 * lost. If this occurs, trigger a GT reset to recover.
                 */
                smp_rmb();
                ret = wait_event_timeout(guc->ct.wq,
                                         !xe_uc_fw_is_running(&guc->fw) ||
                                         !exec_queue_pending_disable(q) ||
                                         xe_guc_read_stopped(guc) ||
                                         vf_recovery(guc), HZ * 5);
                if (vf_recovery(guc))
                        goto handle_vf_resume;
                if (!ret || xe_guc_read_stopped(guc)) {
trigger_reset:
                        if (!ret)
                                xe_gt_warn(guc_to_gt(guc),
                                           "Schedule disable failed to respond, guc_id=%d",
                                           q->guc->id);
                        xe_devcoredump(q, job,
                                       "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d",
                                       q->guc->id, ret, xe_guc_read_stopped(guc));
                        xe_gt_reset_async(q->gt);
                        xe_sched_tdr_queue_imm(sched);
                        goto rearm;
                }
        }

        if (q->vm && q->vm->xef) {
                process_name = q->vm->xef->process_name;
                pid = q->vm->xef->pid;
        }

        if (!exec_queue_killed(q))
                xe_gt_notice(guc_to_gt(guc),
                             "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]",
                             xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
                             q->guc->id, q->flags, process_name, pid);

        trace_xe_sched_job_timedout(job);

        if (!exec_queue_killed(q))
                xe_devcoredump(q, job,
                               "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx",
                               xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
                               q->guc->id, q->flags);

        /*
         * Kernel jobs should never fail, nor should VM jobs if they do
         * somethings has gone wrong and the GT needs a reset
         */
        xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,
                   "Kernel-submitted job timed out\n");
        xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),
                   "VM job timed out on non-killed execqueue\n");
        if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL ||
                        (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) {
                if (!xe_sched_invalidate_job(job, 2)) {
                        xe_gt_reset_async(q->gt);
                        goto rearm;
                }
        }

        /* Mark all outstanding jobs as bad, thus completing them */
        xe_sched_job_set_error(job, err);
        drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL)
                xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED);

        xe_sched_submission_start(sched);

        if (xe_exec_queue_is_multi_queue(q))
                xe_guc_exec_queue_group_trigger_cleanup(q);
        else
                xe_guc_exec_queue_trigger_cleanup(q);

        /*
         * We want the job added back to the pending list so it gets freed; this
         * is what DRM_GPU_SCHED_STAT_NO_HANG does.
         */
        return DRM_GPU_SCHED_STAT_NO_HANG;

rearm:
        /*
         * XXX: Ideally want to adjust timeout based on current execution time
         * but there is not currently an easy way to do in DRM scheduler. With
         * some thought, do this in a follow up.
         */
        xe_sched_submission_start(sched);
handle_vf_resume:
        return DRM_GPU_SCHED_STAT_NO_HANG;
}

static void guc_exec_queue_fini(struct xe_exec_queue *q)
{
        struct xe_guc_exec_queue *ge = q->guc;
        struct xe_guc *guc = exec_queue_to_guc(q);

        release_guc_id(guc, q);
        xe_sched_entity_fini(&ge->entity);
        xe_sched_fini(&ge->sched);

        /*
         * RCU free due sched being exported via DRM scheduler fences
         * (timeline name).
         */
        kfree_rcu(ge, rcu);
}

static void __guc_exec_queue_destroy_async(struct work_struct *w)
{
        struct xe_guc_exec_queue *ge =
                container_of(w, struct xe_guc_exec_queue, destroy_async);
        struct xe_exec_queue *q = ge->q;
        struct xe_guc *guc = exec_queue_to_guc(q);

        guard(xe_pm_runtime)(guc_to_xe(guc));
        trace_xe_exec_queue_destroy(q);

        if (xe_exec_queue_is_multi_queue_secondary(q)) {
                struct xe_exec_queue_group *group = q->multi_queue.group;

                mutex_lock(&group->list_lock);
                list_del(&q->multi_queue.link);
                mutex_unlock(&group->list_lock);
        }

        /* Confirm no work left behind accessing device structures */
        cancel_delayed_work_sync(&ge->sched.base.work_tdr);

        xe_exec_queue_fini(q);
}

static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);

        INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async);

        /* We must block on kernel engines so slabs are empty on driver unload */
        if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q))
                __guc_exec_queue_destroy_async(&q->guc->destroy_async);
        else
                queue_work(xe->destroy_wq, &q->guc->destroy_async);
}

static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q)
{
        /*
         * Might be done from within the GPU scheduler, need to do async as we
         * fini the scheduler when the engine is fini'd, the scheduler can't
         * complete fini within itself (circular dependency). Async resolves
         * this we and don't really care when everything is fini'd, just that it
         * is.
         */
        guc_exec_queue_destroy_async(q);
}

static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg)
{
        struct xe_exec_queue *q = msg->private_data;
        struct xe_guc *guc = exec_queue_to_guc(q);

        xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
        trace_xe_exec_queue_cleanup_entity(q);

        /*
         * Expected state transitions for cleanup:
         * - If the exec queue is registered and GuC firmware is running, we must first
         *   disable scheduling and deregister the queue to ensure proper teardown and
         *   resource release in the GuC, then destroy the exec queue on driver side.
         * - If the GuC is already stopped (e.g., during driver unload or GPU reset),
         *   we cannot expect a response for the deregister request. In this case,
         *   it is safe to directly destroy the exec queue on driver side, as the GuC
         *   will not process further requests and all resources must be cleaned up locally.
         */
        if (exec_queue_registered(q) && xe_uc_fw_is_running(&guc->fw))
                disable_scheduling_deregister(guc, q);
        else
                __guc_exec_queue_destroy(guc, q);
}

static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)
{
        return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q);
}

static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg)
{
        struct xe_exec_queue *q = msg->private_data;
        struct xe_guc *guc = exec_queue_to_guc(q);

        if (guc_exec_queue_allowed_to_change_state(q))
                init_policies(guc, q);
        kfree(msg);
}

static void __suspend_fence_signal(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);

        if (!q->guc->suspend_pending)
                return;

        WRITE_ONCE(q->guc->suspend_pending, false);

        /*
         * We use a GuC shared wait queue for VFs because the VF resfix start
         * interrupt must be able to wake all instances of suspend_wait. This
         * prevents the VF migration worker from being starved during
         * scheduling.
         */
        if (IS_SRIOV_VF(xe))
                wake_up_all(&guc->ct.wq);
        else
                wake_up(&q->guc->suspend_wait);
}

static void suspend_fence_signal(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);

        xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) ||
                     xe_guc_read_stopped(guc));
        xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending);

        __suspend_fence_signal(q);
}

static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
{
        struct xe_exec_queue *q = msg->private_data;
        struct xe_guc *guc = exec_queue_to_guc(q);
        bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);

        if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) &&
            !exec_queue_suspended(q) && exec_queue_enabled(q)) {
                wait_event(guc->ct.wq, vf_recovery(guc) ||
                           ((q->guc->resume_time != RESUME_PENDING ||
                           xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));

                if (!xe_guc_read_stopped(guc)) {
                        s64 since_resume_ms =
                                ktime_ms_delta(ktime_get(),
                                               q->guc->resume_time);
                        s64 wait_ms = q->vm->preempt.min_run_period_ms -
                                since_resume_ms;

                        if (wait_ms > 0 && q->guc->resume_time)
                                relaxed_ms_sleep(wait_ms);

                        set_exec_queue_suspended(q);
                        disable_scheduling(q, false);
                }
        } else if (q->guc->suspend_pending) {
                if (idle_skip_suspend)
                        set_exec_queue_idle_skip_suspend(q);
                set_exec_queue_suspended(q);
                suspend_fence_signal(q);
        }
}

static void sched_context(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_lrc *lrc = q->lrc[0];
        u32 action[] = {
                XE_GUC_ACTION_SCHED_CONTEXT,
                q->guc->id,
        };

        xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));

        trace_xe_exec_queue_submit(q);

        xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
        xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
}

static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
{
        struct xe_exec_queue *q = msg->private_data;

        if (guc_exec_queue_allowed_to_change_state(q)) {
                clear_exec_queue_suspended(q);
                if (!exec_queue_enabled(q)) {
                        if (exec_queue_idle_skip_suspend(q)) {
                                struct xe_lrc *lrc = q->lrc[0];

                                clear_exec_queue_idle_skip_suspend(q);
                                xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
                        }
                        q->guc->resume_time = RESUME_PENDING;
                        set_exec_queue_pending_resume(q);
                        enable_scheduling(q);
                } else if (exec_queue_idle_skip_suspend(q)) {
                        clear_exec_queue_idle_skip_suspend(q);
                        sched_context(q);
                }
        } else {
                clear_exec_queue_suspended(q);
                clear_exec_queue_idle_skip_suspend(q);
        }
}

static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg)
{
        struct xe_exec_queue *q = msg->private_data;

        if (guc_exec_queue_allowed_to_change_state(q)) {
#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE        (2)
                struct xe_guc *guc = exec_queue_to_guc(q);
                struct xe_exec_queue_group *group = q->multi_queue.group;
                u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
                int len = 0;

                action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
                action[len++] = group->primary->guc->id;

                xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE

                xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
        }

        kfree(msg);
}

#define CLEANUP                         1       /* Non-zero values to catch uninitialized msg */
#define SET_SCHED_PROPS                 2
#define SUSPEND                         3
#define RESUME                          4
#define SET_MULTI_QUEUE_PRIORITY        5
#define OPCODE_MASK     0xf
#define MSG_LOCKED      BIT(8)
#define MSG_HEAD        BIT(9)

static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
{
        struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data));

        trace_xe_sched_msg_recv(msg);

        switch (msg->opcode) {
        case CLEANUP:
                __guc_exec_queue_process_msg_cleanup(msg);
                break;
        case SET_SCHED_PROPS:
                __guc_exec_queue_process_msg_set_sched_props(msg);
                break;
        case SUSPEND:
                __guc_exec_queue_process_msg_suspend(msg);
                break;
        case RESUME:
                __guc_exec_queue_process_msg_resume(msg);
                break;
        case SET_MULTI_QUEUE_PRIORITY:
                __guc_exec_queue_process_msg_set_multi_queue_priority(msg);
                break;
        default:
                XE_WARN_ON("Unknown message type");
        }

        xe_pm_runtime_put(xe);
}

static const struct drm_sched_backend_ops drm_sched_ops = {
        .run_job = guc_exec_queue_run_job,
        .free_job = guc_exec_queue_free_job,
        .timedout_job = guc_exec_queue_timedout_job,
};

static const struct xe_sched_backend_ops xe_sched_ops = {
        .process_msg = guc_exec_queue_process_msg,
};

static int guc_exec_queue_init(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched;
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct workqueue_struct *submit_wq = NULL;
        struct xe_guc_exec_queue *ge;
        long timeout;
        int err, i;

        xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc)));

        ge = kzalloc_obj(*ge);
        if (!ge)
                return -ENOMEM;

        q->guc = ge;
        ge->q = q;
        init_rcu_head(&ge->rcu);
        init_waitqueue_head(&ge->suspend_wait);

        for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i)
                INIT_LIST_HEAD(&ge->static_msgs[i].link);

        timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
                  msecs_to_jiffies(q->sched_props.job_timeout_ms);

        /*
         * Use primary queue's submit_wq for all secondary queues of a
         * multi queue group. This serialization avoids any locking around
         * CGP synchronization with GuC.
         */
        if (xe_exec_queue_is_multi_queue_secondary(q)) {
                struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);

                submit_wq = primary->guc->sched.base.submit_wq;
        }

        err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
                            submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
                            timeout, guc_to_gt(guc)->ordered_wq, NULL,
                            q->name, gt_to_xe(q->gt)->drm.dev);
        if (err)
                goto err_free;

        sched = &ge->sched;
        err = xe_sched_entity_init(&ge->entity, sched);
        if (err)
                goto err_sched;

        mutex_lock(&guc->submission_state.lock);

        err = alloc_guc_id(guc, q);
        if (err)
                goto err_entity;

        q->entity = &ge->entity;

        if (xe_guc_read_stopped(guc) || vf_recovery(guc))
                xe_sched_stop(sched);

        mutex_unlock(&guc->submission_state.lock);

        xe_exec_queue_assign_name(q, q->guc->id);

        /*
         * Maintain secondary queues of the multi queue group in a list
         * for handling dependencies across the queues in the group.
         */
        if (xe_exec_queue_is_multi_queue_secondary(q)) {
                struct xe_exec_queue_group *group = q->multi_queue.group;

                INIT_LIST_HEAD(&q->multi_queue.link);
                mutex_lock(&group->list_lock);
                list_add_tail(&q->multi_queue.link, &group->list);
                mutex_unlock(&group->list_lock);
        }

        if (xe_exec_queue_is_multi_queue(q))
                trace_xe_exec_queue_create_multi_queue(q);
        else
                trace_xe_exec_queue_create(q);

        return 0;

err_entity:
        mutex_unlock(&guc->submission_state.lock);
        xe_sched_entity_fini(&ge->entity);
err_sched:
        xe_sched_fini(&ge->sched);
err_free:
        kfree(ge);

        return err;
}

static void guc_exec_queue_kill(struct xe_exec_queue *q)
{
        trace_xe_exec_queue_kill(q);
        set_exec_queue_killed(q);
        __suspend_fence_signal(q);
        xe_guc_exec_queue_trigger_cleanup(q);
}

static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,
                                   u32 opcode)
{
        xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q)));

        INIT_LIST_HEAD(&msg->link);
        msg->opcode = opcode & OPCODE_MASK;
        msg->private_data = q;

        trace_xe_sched_msg_add(msg);
        if (opcode & MSG_HEAD)
                xe_sched_add_msg_head(&q->guc->sched, msg);
        else if (opcode & MSG_LOCKED)
                xe_sched_add_msg_locked(&q->guc->sched, msg);
        else
                xe_sched_add_msg(&q->guc->sched, msg);
}

static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q,
                                            struct xe_sched_msg *msg,
                                            u32 opcode)
{
        if (!list_empty(&msg->link))
                return;

        guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD);
}

static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q,
                                       struct xe_sched_msg *msg,
                                       u32 opcode)
{
        if (!list_empty(&msg->link))
                return false;

        guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED);

        return true;
}

#define STATIC_MSG_CLEANUP      0
#define STATIC_MSG_SUSPEND      1
#define STATIC_MSG_RESUME       2
static void guc_exec_queue_destroy(struct xe_exec_queue *q)
{
        struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;

        if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q))
                guc_exec_queue_add_msg(q, msg, CLEANUP);
        else
                __guc_exec_queue_destroy(exec_queue_to_guc(q), q);
}

static int guc_exec_queue_set_priority(struct xe_exec_queue *q,
                                       enum xe_exec_queue_priority priority)
{
        struct xe_sched_msg *msg;

        if (q->sched_props.priority == priority ||
            exec_queue_killed_or_banned_or_wedged(q))
                return 0;

        msg = kmalloc_obj(*msg);
        if (!msg)
                return -ENOMEM;

        q->sched_props.priority = priority;
        guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);

        return 0;
}

static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us)
{
        struct xe_sched_msg *msg;

        if (q->sched_props.timeslice_us == timeslice_us ||
            exec_queue_killed_or_banned_or_wedged(q))
                return 0;

        msg = kmalloc_obj(*msg);
        if (!msg)
                return -ENOMEM;

        q->sched_props.timeslice_us = timeslice_us;
        guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);

        return 0;
}

static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q,
                                              u32 preempt_timeout_us)
{
        struct xe_sched_msg *msg;

        if (q->sched_props.preempt_timeout_us == preempt_timeout_us ||
            exec_queue_killed_or_banned_or_wedged(q))
                return 0;

        msg = kmalloc_obj(*msg);
        if (!msg)
                return -ENOMEM;

        q->sched_props.preempt_timeout_us = preempt_timeout_us;
        guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);

        return 0;
}

static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
                                                   enum xe_multi_queue_priority priority)
{
        struct xe_sched_msg *msg;

        xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));

        if (q->multi_queue.priority == priority ||
            exec_queue_killed_or_banned_or_wedged(q))
                return 0;

        msg = kmalloc_obj(*msg);
        if (!msg)
                return -ENOMEM;

        q->multi_queue.priority = priority;
        guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);

        return 0;
}

static int guc_exec_queue_suspend(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;

        if (exec_queue_killed_or_banned_or_wedged(q))
                return -EINVAL;

        xe_sched_msg_lock(sched);
        if (guc_exec_queue_try_add_msg(q, msg, SUSPEND))
                q->guc->suspend_pending = true;
        xe_sched_msg_unlock(sched);

        return 0;
}

static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        int ret;

        /*
         * Likely don't need to check exec_queue_killed() as we clear
         * suspend_pending upon kill but to be paranoid but races in which
         * suspend_pending is set after kill also check kill here.
         */
#define WAIT_COND \
        (!READ_ONCE(q->guc->suspend_pending) || exec_queue_killed(q) || \
         xe_guc_read_stopped(guc))

retry:
        if (IS_SRIOV_VF(xe))
                ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND ||
                                                       vf_recovery(guc),
                                                       HZ * 5);
        else
                ret = wait_event_interruptible_timeout(q->guc->suspend_wait,
                                                       WAIT_COND, HZ * 5);

        if (vf_recovery(guc) && !xe_device_wedged((guc_to_xe(guc))))
                return -EAGAIN;

        if (!ret) {
                xe_gt_warn(guc_to_gt(guc),
                           "Suspend fence, guc_id=%d, failed to respond",
                           q->guc->id);
                /* XXX: Trigger GT reset? */
                return -ETIME;
        } else if (IS_SRIOV_VF(xe) && !WAIT_COND) {
                /* Corner case on RESFIX DONE where vf_recovery() changes */
                goto retry;
        }

#undef WAIT_COND

        return ret < 0 ? ret : 0;
}

static void guc_exec_queue_resume(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME;
        struct xe_guc *guc = exec_queue_to_guc(q);

        xe_gt_assert(guc_to_gt(guc), !q->guc->suspend_pending);

        xe_sched_msg_lock(sched);
        guc_exec_queue_try_add_msg(q, msg, RESUME);
        xe_sched_msg_unlock(sched);
}

static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
{
        if (xe_exec_queue_is_multi_queue_secondary(q) &&
            guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)))
                return true;

        return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);
}

/*
 * All of these functions are an abstraction layer which other parts of Xe can
 * use to trap into the GuC backend. All of these functions, aside from init,
 * really shouldn't do much other than trap into the DRM scheduler which
 * synchronizes these operations.
 */
static const struct xe_exec_queue_ops guc_exec_queue_ops = {
        .init = guc_exec_queue_init,
        .kill = guc_exec_queue_kill,
        .fini = guc_exec_queue_fini,
        .destroy = guc_exec_queue_destroy,
        .set_priority = guc_exec_queue_set_priority,
        .set_timeslice = guc_exec_queue_set_timeslice,
        .set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
        .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
        .suspend = guc_exec_queue_suspend,
        .suspend_wait = guc_exec_queue_suspend_wait,
        .resume = guc_exec_queue_resume,
        .reset_status = guc_exec_queue_reset_status,
};

static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        bool do_destroy = false;

        /* Stop scheduling + flush any DRM scheduler operations */
        xe_sched_submission_stop(sched);

        /* Clean up lost G2H + reset engine state */
        if (exec_queue_registered(q)) {
                if (exec_queue_destroyed(q))
                        do_destroy = true;
        }
        if (q->guc->suspend_pending) {
                set_exec_queue_suspended(q);
                suspend_fence_signal(q);
        }
        atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED |
                   EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED |
                   EXEC_QUEUE_STATE_SUSPENDED,
                   &q->guc->state);
        q->guc->resume_time = 0;
        trace_xe_exec_queue_stop(q);

        /*
         * Ban any engine (aside from kernel and engines used for VM ops) with a
         * started but not complete job or if a job has gone through a GT reset
         * more than twice.
         */
        if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) {
                struct xe_sched_job *job = xe_sched_first_pending_job(sched);
                bool ban = false;

                if (job) {
                        if ((xe_sched_job_started(job) &&
                            !xe_sched_job_completed(job)) ||
                            xe_sched_invalidate_job(job, 2)) {
                                trace_xe_sched_job_ban(job);
                                ban = true;
                        }
                }

                if (ban) {
                        set_exec_queue_banned(q);
                        xe_guc_exec_queue_trigger_cleanup(q);
                }
        }

        if (do_destroy)
                __guc_exec_queue_destroy(guc, q);
}

static int guc_submit_reset_prepare(struct xe_guc *guc)
{
        int ret;

        /*
         * Using an atomic here rather than submission_state.lock as this
         * function can be called while holding the CT lock (engine reset
         * failure). submission_state.lock needs the CT lock to resubmit jobs.
         * Atomic is not ideal, but it works to prevent against concurrent reset
         * and releasing any TDRs waiting on guc->submission_state.stopped.
         */
        ret = atomic_fetch_or(1, &guc->submission_state.stopped);
        smp_wmb();
        wake_up_all(&guc->ct.wq);

        return ret;
}

int xe_guc_submit_reset_prepare(struct xe_guc *guc)
{
        if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
                return 0;

        if (!guc->submission_state.initialized)
                return 0;

        return guc_submit_reset_prepare(guc);
}

void xe_guc_submit_reset_wait(struct xe_guc *guc)
{
        wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) ||
                   !xe_guc_read_stopped(guc));
}

void xe_guc_submit_stop(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1);

        mutex_lock(&guc->submission_state.lock);

        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /* Prevent redundant attempts to stop parallel queues */
                if (q->guc->id != index)
                        continue;

                guc_exec_queue_stop(guc, q);
        }

        mutex_unlock(&guc->submission_state.lock);

        /*
         * No one can enter the backend at this point, aside from new engine
         * creation which is protected by guc->submission_state.lock.
         */

}

static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc,
                                                       struct xe_exec_queue *q)
{
        bool pending_enable, pending_disable, pending_resume;

        pending_enable = exec_queue_pending_enable(q);
        pending_resume = exec_queue_pending_resume(q);

        if (pending_enable && pending_resume) {
                q->guc->needs_resume = true;
                xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d",
                          q->guc->id);
        }

        if (pending_enable && !pending_resume) {
                clear_exec_queue_registered(q);
                xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d",
                          q->guc->id);
        }

        if (pending_enable) {
                clear_exec_queue_enabled(q);
                clear_exec_queue_pending_resume(q);
                clear_exec_queue_pending_enable(q);
                xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d",
                          q->guc->id);
        }

        if (exec_queue_destroyed(q) && exec_queue_registered(q)) {
                clear_exec_queue_destroyed(q);
                q->guc->needs_cleanup = true;
                xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d",
                          q->guc->id);
        }

        pending_disable = exec_queue_pending_disable(q);

        if (pending_disable && exec_queue_suspended(q)) {
                clear_exec_queue_suspended(q);
                q->guc->needs_suspend = true;
                xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d",
                          q->guc->id);
        }

        if (pending_disable) {
                if (!pending_enable)
                        set_exec_queue_enabled(q);
                clear_exec_queue_pending_disable(q);
                xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d",
                          q->guc->id);
        }

        q->guc->resume_time = 0;
}

static void lrc_parallel_clear(struct xe_lrc *lrc)
{
        struct xe_device *xe = gt_to_xe(lrc->gt);
        struct iosys_map map = xe_lrc_parallel_map(lrc);
        int i;

        for (i = 0; i < WQ_SIZE / sizeof(u32); ++i)
                parallel_write(xe, map, wq[i],
                               FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
                               FIELD_PREP(WQ_LEN_MASK, 0));
}

/*
 * This function is quite complex but only real way to ensure no state is lost
 * during VF resume flows. The function scans the queue state, make adjustments
 * as needed, and queues jobs / messages which replayed upon unpause.
 */
static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_sched_job *job;
        int i;

        lockdep_assert_held(&guc->submission_state.lock);

        /* Stop scheduling + flush any DRM scheduler operations */
        xe_sched_submission_stop(sched);
        cancel_delayed_work_sync(&sched->base.work_tdr);

        guc_exec_queue_revert_pending_state_change(guc, q);

        if (xe_exec_queue_is_parallel(q)) {
                /* Pairs with WRITE_ONCE in __xe_exec_queue_init  */
                struct xe_lrc *lrc = READ_ONCE(q->lrc[0]);

                /*
                 * NOP existing WQ commands that may contain stale GGTT
                 * addresses. These will be replayed upon unpause. The hardware
                 * seems to get confused if the WQ head/tail pointers are
                 * adjusted.
                 */
                if (lrc)
                        lrc_parallel_clear(lrc);
        }

        job = xe_sched_first_pending_job(sched);
        if (job) {
                job->restore_replay = true;

                /*
                 * Adjust software tail so jobs submitted overwrite previous
                 * position in ring buffer with new GGTT addresses.
                 */
                for (i = 0; i < q->width; ++i)
                        q->lrc[i]->ring.tail = job->ptrs[i].head;
        }
}

/**
 * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC.
 * @guc: the &xe_guc struct instance whose scheduler is to be disabled
 */
void xe_guc_submit_pause(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
                xe_sched_submission_stop(&q->guc->sched);
        mutex_unlock(&guc->submission_state.lock);
}

/**
 * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF.
 * @guc: the &xe_guc struct instance whose scheduler is to be disabled
 */
void xe_guc_submit_pause_vf(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
        xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /* Prevent redundant attempts to stop parallel queues */
                if (q->guc->id != index)
                        continue;

                guc_exec_queue_pause(guc, q);
        }
        mutex_unlock(&guc->submission_state.lock);
}

static void guc_exec_queue_start(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;

        if (!exec_queue_killed_or_banned_or_wedged(q)) {
                struct xe_sched_job *job = xe_sched_first_pending_job(sched);
                int i;

                trace_xe_exec_queue_resubmit(q);
                if (job) {
                        for (i = 0; i < q->width; ++i) {
                                /*
                                 * The GuC context is unregistered at this point
                                 * time, adjusting software ring tail ensures
                                 * jobs are rewritten in original placement,
                                 * adjusting LRC tail ensures the newly loaded
                                 * GuC / contexts only view the LRC tail
                                 * increasing as jobs are written out.
                                 */
                                q->lrc[i]->ring.tail = job->ptrs[i].head;
                                xe_lrc_set_ring_tail(q->lrc[i],
                                                     xe_lrc_ring_head(q->lrc[i]));
                        }
                }
                xe_sched_resubmit_jobs(sched);
        }

        xe_sched_submission_start(sched);
        xe_sched_submission_resume_tdr(sched);
}

int xe_guc_submit_start(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1);

        mutex_lock(&guc->submission_state.lock);
        atomic_dec(&guc->submission_state.stopped);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /* Prevent redundant attempts to start parallel queues */
                if (q->guc->id != index)
                        continue;

                guc_exec_queue_start(q);
        }
        mutex_unlock(&guc->submission_state.lock);

        wake_up_all(&guc->ct.wq);

        return 0;
}

static void guc_exec_queue_unpause_prepare(struct xe_guc *guc,
                                           struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_sched_job *job = NULL;
        struct drm_sched_job *s_job;
        bool restore_replay = false;

        drm_sched_for_each_pending_job(s_job, &sched->base, NULL) {
                job = to_xe_sched_job(s_job);
                restore_replay |= job->restore_replay;
                if (restore_replay) {
                        xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d",
                                  q->guc->id, xe_sched_job_seqno(job));

                        q->ring_ops->emit_job(job);
                        job->restore_replay = true;
                }
        }

        if (job)
                job->last_replay = true;
}

/**
 * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF.
 * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause
 */
void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
        xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /* Prevent redundant attempts to stop parallel queues */
                if (q->guc->id != index)
                        continue;

                guc_exec_queue_unpause_prepare(guc, q);
        }
        mutex_unlock(&guc->submission_state.lock);
}

static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_sched_msg *msg;

        if (q->guc->needs_cleanup) {
                msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;

                guc_exec_queue_add_msg(q, msg, CLEANUP);
                q->guc->needs_cleanup = false;
        }

        if (q->guc->needs_suspend) {
                msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;

                xe_sched_msg_lock(sched);
                guc_exec_queue_try_add_msg_head(q, msg, SUSPEND);
                xe_sched_msg_unlock(sched);

                q->guc->needs_suspend = false;
        }

        /*
         * The resume must be in the message queue before the suspend as it is
         * not possible for a resume to be issued if a suspend pending is, but
         * the inverse is possible.
         */
        if (q->guc->needs_resume) {
                msg = q->guc->static_msgs + STATIC_MSG_RESUME;

                xe_sched_msg_lock(sched);
                guc_exec_queue_try_add_msg_head(q, msg, RESUME);
                xe_sched_msg_unlock(sched);

                q->guc->needs_resume = false;
        }
}

static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q);

        lockdep_assert_held(&guc->submission_state.lock);

        xe_sched_resubmit_jobs(sched);
        guc_exec_queue_replay_pending_state_change(q);
        xe_sched_submission_start(sched);
        if (needs_tdr)
                xe_guc_exec_queue_trigger_cleanup(q);
        xe_sched_submission_resume_tdr(sched);
}

/**
 * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC.
 * @guc: the &xe_guc struct instance whose scheduler is to be enabled
 */
void xe_guc_submit_unpause(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
                xe_sched_submission_start(&q->guc->sched);
        mutex_unlock(&guc->submission_state.lock);
}

/**
 * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF.
 * @guc: the &xe_guc struct instance whose scheduler is to be enabled
 */
void xe_guc_submit_unpause_vf(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /*
                 * Prevent redundant attempts to stop parallel queues, or queues
                 * created after resfix done.
                 */
                if (q->guc->id != index ||
                    !drm_sched_is_stopped(&q->guc->sched.base))
                        continue;

                guc_exec_queue_unpause(guc, q);
        }
        mutex_unlock(&guc->submission_state.lock);
}

/**
 * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC.
 * @guc: the &xe_guc struct instance whose scheduler is to be aborted
 */
void xe_guc_submit_pause_abort(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                struct xe_gpu_scheduler *sched = &q->guc->sched;

                /* Prevent redundant attempts to stop parallel queues */
                if (q->guc->id != index)
                        continue;

                xe_sched_submission_start(sched);
                guc_exec_queue_kill(q);
        }
        mutex_unlock(&guc->submission_state.lock);
}

static struct xe_exec_queue *
g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id)
{
        struct xe_gt *gt = guc_to_gt(guc);
        struct xe_exec_queue *q;

        if (unlikely(guc_id >= GUC_ID_MAX)) {
                xe_gt_err(gt, "Invalid guc_id %u\n", guc_id);
                return NULL;
        }

        q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id);
        if (unlikely(!q)) {
                xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id);
                return NULL;
        }

        xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id);
        xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width));

        return q;
}

static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
{
        u32 action[] = {
                XE_GUC_ACTION_DEREGISTER_CONTEXT,
                q->guc->id,
        };

        xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q));
        xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
        xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));

        trace_xe_exec_queue_deregister(q);

        if (xe_exec_queue_is_multi_queue_secondary(q))
                handle_deregister_done(guc, q);
        else
                xe_guc_ct_send_g2h_handler(&guc->ct, action,
                                           ARRAY_SIZE(action));
}

static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q,
                              u32 runnable_state)
{
        trace_xe_exec_queue_scheduling_done(q);

        if (runnable_state == 1) {
                xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q));

                q->guc->resume_time = ktime_get();
                clear_exec_queue_pending_resume(q);
                clear_exec_queue_pending_enable(q);
                smp_wmb();
                wake_up_all(&guc->ct.wq);
        } else {
                xe_gt_assert(guc_to_gt(guc), runnable_state == 0);
                xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q));

                if (q->guc->suspend_pending) {
                        suspend_fence_signal(q);
                        clear_exec_queue_pending_disable(q);
                } else {
                        if (exec_queue_banned(q)) {
                                smp_wmb();
                                wake_up_all(&guc->ct.wq);
                        }
                        if (exec_queue_destroyed(q)) {
                                /*
                                 * Make sure to clear the pending_disable only
                                 * after sampling the destroyed state. We want
                                 * to ensure we don't trigger the unregister too
                                 * early with something intending to only
                                 * disable scheduling. The caller doing the
                                 * destroy must wait for an ongoing
                                 * pending_disable before marking as destroyed.
                                 */
                                clear_exec_queue_pending_disable(q);
                                deregister_exec_queue(guc, q);
                        } else {
                                clear_exec_queue_pending_disable(q);
                        }
                }
        }
}

static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
                                                    struct xe_exec_queue *q,
                                                    u32 runnable_state)
{
        /* Take CT lock here as handle_sched_done() do send a h2g message */
        mutex_lock(&guc->ct.lock);
        handle_sched_done(guc, q, runnable_state);
        mutex_unlock(&guc->ct.lock);
}

int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        struct xe_exec_queue *q;
        u32 guc_id, runnable_state;

        if (unlikely(len < 2))
                return -EPROTO;

        guc_id = msg[0];
        runnable_state = msg[1];

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        if (unlikely(!exec_queue_pending_enable(q) &&
                     !exec_queue_pending_disable(q))) {
                xe_gt_err(guc_to_gt(guc),
                          "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u",
                          atomic_read(&q->guc->state), q->guc->id,
                          runnable_state);
                return -EPROTO;
        }

        handle_sched_done(guc, q, runnable_state);

        return 0;
}

static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)
{
        trace_xe_exec_queue_deregister_done(q);

        clear_exec_queue_registered(q);
        __guc_exec_queue_destroy(guc, q);
}

int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        struct xe_exec_queue *q;
        u32 guc_id;

        if (unlikely(len < 1))
                return -EPROTO;

        guc_id = msg[0];

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||
            exec_queue_pending_enable(q) || exec_queue_enabled(q)) {
                xe_gt_err(guc_to_gt(guc),
                          "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d",
                          atomic_read(&q->guc->state), q->guc->id);
                return -EPROTO;
        }

        handle_deregister_done(guc, q);

        return 0;
}

int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        struct xe_gt *gt = guc_to_gt(guc);
        struct xe_exec_queue *q;
        u32 guc_id;

        if (unlikely(len < 1))
                return -EPROTO;

        guc_id = msg[0];

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
                   xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
                   atomic_read(&q->guc->state));

        trace_xe_exec_queue_reset(q);

        /*
         * A banned engine is a NOP at this point (came from
         * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel
         * jobs by setting timeout of the job to the minimum value kicking
         * guc_exec_queue_timedout_job.
         */
        xe_guc_exec_queue_reset_trigger_cleanup(q);

        return 0;
}

/*
 * xe_guc_error_capture_handler - Handler of GuC captured message
 * @guc: The GuC object
 * @msg: Point to the message
 * @len: The message length
 *
 * When GuC captured data is ready, GuC will send message
 * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be
 * called 1st to check status before process the data comes with the message.
 *
 * Returns: error code. 0 if success
 */
int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        u32 status;

        if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN))
                return -EPROTO;

        status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;
        if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
                xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space");

        xe_guc_capture_process(guc);

        return 0;
}

int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
                                               u32 len)
{
        struct xe_gt *gt = guc_to_gt(guc);
        struct xe_exec_queue *q;
        u32 guc_id;
        u32 type = XE_GUC_CAT_ERR_TYPE_INVALID;

        if (unlikely(!len || len > 2))
                return -EPROTO;

        guc_id = msg[0];

        if (len == 2)
                type = msg[1];

        if (guc_id == GUC_ID_UNKNOWN) {
                /*
                 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF
                 * context. In such case only PF will be notified about that fault.
                 */
                xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n");
                return 0;
        }

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        /*
         * The type is HW-defined and changes based on platform, so we don't
         * decode it in the kernel and only check if it is valid.
         * See bspec 54047 and 72187 for details.
         */
        if (type != XE_GUC_CAT_ERR_TYPE_INVALID)
                xe_gt_info(gt,
                           "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",
                           type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
        else
                xe_gt_info(gt,
                           "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",
                           xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);

        trace_xe_exec_queue_memory_cat_error(q);

        /* Treat the same as engine reset */
        xe_guc_exec_queue_reset_trigger_cleanup(q);

        return 0;
}

int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        struct xe_gt *gt = guc_to_gt(guc);
        u8 guc_class, instance;
        u32 reason;

        if (unlikely(len != 3))
                return -EPROTO;

        guc_class = msg[0];
        instance = msg[1];
        reason = msg[2];

        /* Unexpected failure of a hardware feature, log an actual error */
        xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X",
                  guc_class, instance, reason);

        xe_gt_reset_async(gt);

        return 0;
}

int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
                                                u32 len)
{
        struct xe_gt *gt = guc_to_gt(guc);
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_exec_queue *q;
        u32 guc_id = msg[2];

        if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) {
                drm_err(&xe->drm, "Invalid length %u", len);
                return -EPROTO;
        }

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        xe_gt_dbg(gt,
                  "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x",
                  msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]);

        trace_xe_exec_queue_cgp_context_error(q);

        /* Treat the same as engine reset */
        xe_guc_exec_queue_reset_trigger_cleanup(q);

        return 0;
}

/**
 * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler
 * @guc: guc
 * @msg: message indicating CGP sync done
 * @len: length of message
 *
 * Set multi queue group's sync_pending flag to false and wakeup anyone waiting
 * for CGP synchronization to complete.
 *
 * Return: 0 on success, -EPROTO for malformed messages.
 */
int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_exec_queue *q;
        u32 guc_id = msg[0];

        if (unlikely(len < 1)) {
                drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len);
                return -EPROTO;
        }

        q = g2h_exec_queue_lookup(guc, guc_id);
        if (unlikely(!q))
                return -EPROTO;

        if (!xe_exec_queue_is_multi_queue_primary(q)) {
                drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response");
                return -EPROTO;
        }

        /* Wakeup the serialized cgp update wait */
        WRITE_ONCE(q->multi_queue.group->sync_pending, false);
        xe_guc_ct_wake_waiters(&guc->ct);

        return 0;
}

static void
guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
                                   struct xe_guc_submit_exec_queue_snapshot *snapshot)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
        int i;

        snapshot->guc.wqi_head = q->guc->wqi_head;
        snapshot->guc.wqi_tail = q->guc->wqi_tail;
        snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head);
        snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail);
        snapshot->parallel.wq_desc.status = parallel_read(xe, map,
                                                          wq_desc.wq_status);

        if (snapshot->parallel.wq_desc.head !=
            snapshot->parallel.wq_desc.tail) {
                for (i = snapshot->parallel.wq_desc.head;
                     i != snapshot->parallel.wq_desc.tail;
                     i = (i + sizeof(u32)) % WQ_SIZE)
                        snapshot->parallel.wq[i / sizeof(u32)] =
                                parallel_read(xe, map, wq[i / sizeof(u32)]);
        }
}

static void
guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
                                 struct drm_printer *p)
{
        int i;

        drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n",
                   snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head);
        drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n",
                   snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail);
        drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status);

        if (snapshot->parallel.wq_desc.head !=
            snapshot->parallel.wq_desc.tail) {
                for (i = snapshot->parallel.wq_desc.head;
                     i != snapshot->parallel.wq_desc.tail;
                     i = (i + sizeof(u32)) % WQ_SIZE)
                        drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32),
                                   snapshot->parallel.wq[i / sizeof(u32)]);
        }
}

/**
 * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine.
 * @q: faulty exec queue
 *
 * This can be printed out in a later stage like during dev_coredump
 * analysis.
 *
 * Returns: a GuC Submit Engine snapshot object that must be freed by the
 * caller, using `xe_guc_exec_queue_snapshot_free`.
 */
struct xe_guc_submit_exec_queue_snapshot *
xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
{
        struct xe_gpu_scheduler *sched = &q->guc->sched;
        struct xe_guc_submit_exec_queue_snapshot *snapshot;
        int i;

        snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC);

        if (!snapshot)
                return NULL;

        snapshot->guc.id = q->guc->id;
        memcpy(&snapshot->name, &q->name, sizeof(snapshot->name));
        snapshot->class = q->class;
        snapshot->logical_mask = q->logical_mask;
        snapshot->width = q->width;
        snapshot->refcount = kref_read(&q->refcount);
        snapshot->sched_timeout = sched->base.timeout;
        snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;
        snapshot->sched_props.preempt_timeout_us =
                q->sched_props.preempt_timeout_us;

        snapshot->lrc = kmalloc_objs(struct xe_lrc_snapshot *, q->width,
                                     GFP_ATOMIC);

        if (snapshot->lrc) {
                for (i = 0; i < q->width; ++i) {
                        struct xe_lrc *lrc = q->lrc[i];

                        snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc);
                }
        }

        snapshot->schedule_state = atomic_read(&q->guc->state);
        snapshot->exec_queue_flags = q->flags;

        snapshot->parallel_execution = xe_exec_queue_is_parallel(q);
        if (snapshot->parallel_execution)
                guc_exec_queue_wq_snapshot_capture(q, snapshot);

        if (xe_exec_queue_is_multi_queue(q)) {
                snapshot->multi_queue.valid = true;
                snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
                snapshot->multi_queue.pos = q->multi_queue.pos;
        }

        return snapshot;
}

/**
 * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine.
 * @snapshot: Previously captured snapshot of job.
 *
 * This captures some data that requires taking some locks, so it cannot be done in signaling path.
 */
void
xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot)
{
        int i;

        if (!snapshot || !snapshot->lrc)
                return;

        for (i = 0; i < snapshot->width; ++i)
                xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]);
}

/**
 * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot.
 * @snapshot: GuC Submit Engine snapshot object.
 * @p: drm_printer where it will be printed out.
 *
 * This function prints out a given GuC Submit Engine snapshot object.
 */
void
xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
                                 struct drm_printer *p)
{
        int i;

        if (!snapshot)
                return;

        drm_printf(p, "GuC ID: %d\n", snapshot->guc.id);
        drm_printf(p, "\tName: %s\n", snapshot->name);
        drm_printf(p, "\tClass: %d\n", snapshot->class);
        drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);
        drm_printf(p, "\tWidth: %d\n", snapshot->width);
        drm_printf(p, "\tRef: %d\n", snapshot->refcount);
        drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);
        drm_printf(p, "\tTimeslice: %u (us)\n",
                   snapshot->sched_props.timeslice_us);
        drm_printf(p, "\tPreempt timeout: %u (us)\n",
                   snapshot->sched_props.preempt_timeout_us);

        for (i = 0; snapshot->lrc && i < snapshot->width; ++i)
                xe_lrc_snapshot_print(snapshot->lrc[i], p);

        drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state);
        drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags);

        if (snapshot->parallel_execution)
                guc_exec_queue_wq_snapshot_print(snapshot, p);

        if (snapshot->multi_queue.valid) {
                drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary);
                drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos);
        }
}

/**
 * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given
 * snapshot.
 * @snapshot: GuC Submit Engine snapshot object.
 *
 * This function free all the memory that needed to be allocated at capture
 * time.
 */
void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot)
{
        int i;

        if (!snapshot)
                return;

        if (snapshot->lrc) {
                for (i = 0; i < snapshot->width; i++)
                        xe_lrc_snapshot_free(snapshot->lrc[i]);
                kfree(snapshot->lrc);
        }
        kfree(snapshot);
}

static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p)
{
        struct xe_guc_submit_exec_queue_snapshot *snapshot;

        snapshot = xe_guc_exec_queue_snapshot_capture(q);
        xe_guc_exec_queue_snapshot_print(snapshot, p);
        xe_guc_exec_queue_snapshot_free(snapshot);
}

/**
 * xe_guc_register_vf_exec_queue - Register exec queue for a given context type.
 * @q: Execution queue
 * @ctx_type: Type of the context
 *
 * This function registers the execution queue with the guc. Special context
 * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE
 * are only applicable for IGPU and in the VF.
 * Submits the execution queue to GUC after registering it.
 *
 * Returns - None.
 */
void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type)
{
        struct xe_guc *guc = exec_queue_to_guc(q);
        struct xe_device *xe = guc_to_xe(guc);
        struct xe_gt *gt = guc_to_gt(guc);

        xe_gt_assert(gt, IS_SRIOV_VF(xe));
        xe_gt_assert(gt, !IS_DGFX(xe));
        xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE ||
                     ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE);
        xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0));

        register_exec_queue(q, ctx_type);
        enable_scheduling(q);
}

/**
 * xe_guc_submit_print - GuC Submit Print.
 * @guc: GuC.
 * @p: drm_printer where it will be printed out.
 *
 * This function capture and prints snapshots of **all** GuC Engines.
 */
void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p)
{
        struct xe_exec_queue *q;
        unsigned long index;

        if (!xe_device_uc_enabled(guc_to_xe(guc)))
                return;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
                guc_exec_queue_print(q, p);
        mutex_unlock(&guc->submission_state.lock);
}

/**
 * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues
 * registered with the GuC
 * @guc: GuC.
 *
 * Return: true if any MLRC queue is registered with the GuC, false otherwise.
 */
bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc)
{
        struct xe_exec_queue *q;
        unsigned long index;

        guard(mutex)(&guc->submission_state.lock);

        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
                if (q->width > 1)
                        return true;

        return false;
}

/**
 * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all
 * exec queues registered to given GuC.
 * @guc: the &xe_guc struct instance
 * @scratch: scratch buffer to be used as temporary storage
 *
 * Returns: zero on success, negative error code on failure.
 */
int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch)
{
        struct xe_exec_queue *q;
        unsigned long index;
        int err = 0;

        mutex_lock(&guc->submission_state.lock);
        xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
                /* Prevent redundant attempts to stop parallel queues */
                if (q->guc->id != index)
                        continue;

                err = xe_exec_queue_contexts_hwsp_rebase(q, scratch);
                if (err)
                        break;
        }
        mutex_unlock(&guc->submission_state.lock);

        return err;
}