// SPDX-License-Identifier: MIT
/*
 * Copyright © 2018 Intel Corporation
 */

#include <linux/crc32.h>

#include "gem/i915_gem_stolen.h"

#include "i915_memcpy.h"
#include "i915_selftest.h"
#include "intel_gpu_commands.h"
#include "selftests/igt_reset.h"
#include "selftests/igt_atomic.h"
#include "selftests/igt_spinner.h"

static int
__igt_reset_stolen(struct intel_gt *gt,
                   intel_engine_mask_t mask,
                   const char *msg)
{
        struct i915_ggtt *ggtt = gt->ggtt;
        const struct resource *dsm = &gt->i915->dsm.stolen;
        resource_size_t num_pages, page;
        struct intel_engine_cs *engine;
        intel_wakeref_t wakeref;
        enum intel_engine_id id;
        struct igt_spinner spin;
        long max, count;
        void *tmp;
        u32 *crc;
        int err;

        if (!drm_mm_node_allocated(&ggtt->error_capture))
                return 0;

        num_pages = resource_size(dsm) >> PAGE_SHIFT;
        if (!num_pages)
                return 0;

        crc = kmalloc_array(num_pages, sizeof(u32), GFP_KERNEL);
        if (!crc)
                return -ENOMEM;

        tmp = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (!tmp) {
                err = -ENOMEM;
                goto err_crc;
        }

        igt_global_reset_lock(gt);
        wakeref = intel_runtime_pm_get(gt->uncore->rpm);

        err = igt_spinner_init(&spin, gt);
        if (err)
                goto err_lock;

        for_each_engine(engine, gt, id) {
                struct intel_context *ce;
                struct i915_request *rq;

                if (!(mask & engine->mask))
                        continue;

                if (!intel_engine_can_store_dword(engine))
                        continue;

                ce = intel_context_create(engine);
                if (IS_ERR(ce)) {
                        err = PTR_ERR(ce);
                        goto err_spin;
                }
                rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
                intel_context_put(ce);
                if (IS_ERR(rq)) {
                        err = PTR_ERR(rq);
                        goto err_spin;
                }
                i915_request_add(rq);
        }

        for (page = 0; page < num_pages; page++) {
                dma_addr_t dma = (dma_addr_t)dsm->start + (page << PAGE_SHIFT);
                void __iomem *s;
                void *in;

                ggtt->vm.insert_page(&ggtt->vm, dma,
                                     ggtt->error_capture.start,
                                     i915_gem_get_pat_index(gt->i915,
                                                            I915_CACHE_NONE),
                                     0);
                mb();

                s = io_mapping_map_wc(&ggtt->iomap,
                                      ggtt->error_capture.start,
                                      PAGE_SIZE);

                if (!__drm_mm_interval_first(&gt->i915->mm.stolen,
                                             page << PAGE_SHIFT,
                                             ((page + 1) << PAGE_SHIFT) - 1))
                        memset_io(s, STACK_MAGIC, PAGE_SIZE);

                in = (void __force *)s;
                if (i915_memcpy_from_wc(tmp, in, PAGE_SIZE))
                        in = tmp;
                crc[page] = crc32_le(0, in, PAGE_SIZE);

                io_mapping_unmap(s);
        }
        mb();
        ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE);

        if (mask == ALL_ENGINES) {
                intel_gt_reset(gt, mask, NULL);
        } else {
                for_each_engine(engine, gt, id) {
                        if (mask & engine->mask)
                                intel_engine_reset(engine, NULL);
                }
        }

        max = -1;
        count = 0;
        for (page = 0; page < num_pages; page++) {
                dma_addr_t dma = (dma_addr_t)dsm->start + (page << PAGE_SHIFT);
                void __iomem *s;
                void *in;
                u32 x;

                ggtt->vm.insert_page(&ggtt->vm, dma,
                                     ggtt->error_capture.start,
                                     i915_gem_get_pat_index(gt->i915,
                                                            I915_CACHE_NONE),
                                     0);
                mb();

                s = io_mapping_map_wc(&ggtt->iomap,
                                      ggtt->error_capture.start,
                                      PAGE_SIZE);

                in = (void __force *)s;
                if (i915_memcpy_from_wc(tmp, in, PAGE_SIZE))
                        in = tmp;
                x = crc32_le(0, in, PAGE_SIZE);

                if (x != crc[page] &&
                    !__drm_mm_interval_first(&gt->i915->mm.stolen,
                                             page << PAGE_SHIFT,
                                             ((page + 1) << PAGE_SHIFT) - 1)) {
                        pr_debug("unused stolen page %pa modified by GPU reset\n",
                                 &page);
                        if (count++ == 0)
                                igt_hexdump(in, PAGE_SIZE);
                        max = page;
                }

                io_mapping_unmap(s);
        }
        mb();
        ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE);

        if (count > 0) {
                pr_info("%s reset clobbered %ld pages of stolen, last clobber at page %ld\n",
                        msg, count, max);
        }
        if (max >= I915_GEM_STOLEN_BIAS >> PAGE_SHIFT) {
                pr_err("%s reset clobbered unreserved area [above %x] of stolen; may cause severe faults\n",
                       msg, I915_GEM_STOLEN_BIAS);
                err = -EINVAL;
        }

err_spin:
        igt_spinner_fini(&spin);

err_lock:
        intel_runtime_pm_put(gt->uncore->rpm, wakeref);
        igt_global_reset_unlock(gt);

        kfree(tmp);
err_crc:
        kfree(crc);
        return err;
}

static int igt_reset_device_stolen(void *arg)
{
        return __igt_reset_stolen(arg, ALL_ENGINES, "device");
}

static int igt_reset_engines_stolen(void *arg)
{
        struct intel_gt *gt = arg;
        struct intel_engine_cs *engine;
        enum intel_engine_id id;
        int err;

        if (!intel_has_reset_engine(gt))
                return 0;

        for_each_engine(engine, gt, id) {
                err = __igt_reset_stolen(gt, engine->mask, engine->name);
                if (err)
                        return err;
        }

        return 0;
}

static int igt_global_reset(void *arg)
{
        struct intel_gt *gt = arg;
        unsigned int reset_count;
        intel_wakeref_t wakeref;
        int err = 0;

        /* Check that we can issue a global GPU reset */

        igt_global_reset_lock(gt);
        wakeref = intel_runtime_pm_get(gt->uncore->rpm);

        reset_count = i915_reset_count(&gt->i915->gpu_error);

        intel_gt_reset(gt, ALL_ENGINES, NULL);

        if (i915_reset_count(&gt->i915->gpu_error) == reset_count) {
                pr_err("No GPU reset recorded!\n");
                err = -EINVAL;
        }

        intel_runtime_pm_put(gt->uncore->rpm, wakeref);
        igt_global_reset_unlock(gt);

        if (intel_gt_is_wedged(gt))
                err = -EIO;

        return err;
}

static int igt_wedged_reset(void *arg)
{
        struct intel_gt *gt = arg;
        intel_wakeref_t wakeref;

        /* Check that we can recover a wedged device with a GPU reset */

        igt_global_reset_lock(gt);
        wakeref = intel_runtime_pm_get(gt->uncore->rpm);

        intel_gt_set_wedged(gt);

        GEM_BUG_ON(!intel_gt_is_wedged(gt));
        intel_gt_reset(gt, ALL_ENGINES, NULL);

        intel_runtime_pm_put(gt->uncore->rpm, wakeref);
        igt_global_reset_unlock(gt);

        return intel_gt_is_wedged(gt) ? -EIO : 0;
}

static int igt_atomic_reset(void *arg)
{
        struct intel_gt *gt = arg;
        const typeof(*igt_atomic_phases) *p;
        intel_wakeref_t wakeref;
        int err = 0;

        /* Check that the resets are usable from atomic context */

        wakeref = intel_gt_pm_get(gt);
        igt_global_reset_lock(gt);

        /* Flush any requests before we get started and check basics */
        if (!igt_force_reset(gt))
                goto unlock;

        for (p = igt_atomic_phases; p->name; p++) {
                intel_engine_mask_t awake;

                GEM_TRACE("__intel_gt_reset under %s\n", p->name);

                awake = reset_prepare(gt);
                p->critical_section_begin();

                err = intel_gt_reset_all_engines(gt);

                p->critical_section_end();
                reset_finish(gt, awake);

                if (err) {
                        pr_err("__intel_gt_reset failed under %s\n", p->name);
                        break;
                }
        }

        /* As we poke around the guts, do a full reset before continuing. */
        igt_force_reset(gt);

unlock:
        igt_global_reset_unlock(gt);
        intel_gt_pm_put(gt, wakeref);

        return err;
}

static int igt_atomic_engine_reset(void *arg)
{
        struct intel_gt *gt = arg;
        const typeof(*igt_atomic_phases) *p;
        struct intel_engine_cs *engine;
        enum intel_engine_id id;
        intel_wakeref_t wakeref;
        int err = 0;

        /* Check that the resets are usable from atomic context */

        if (!intel_has_reset_engine(gt))
                return 0;

        if (intel_uc_uses_guc_submission(&gt->uc))
                return 0;

        wakeref = intel_gt_pm_get(gt);
        igt_global_reset_lock(gt);

        /* Flush any requests before we get started and check basics */
        if (!igt_force_reset(gt))
                goto out_unlock;

        for_each_engine(engine, gt, id) {
                struct tasklet_struct *t = &engine->sched_engine->tasklet;

                if (t->func)
                        tasklet_disable(t);
                intel_engine_pm_get(engine);

                for (p = igt_atomic_phases; p->name; p++) {
                        GEM_TRACE("intel_engine_reset(%s) under %s\n",
                                  engine->name, p->name);
                        if (strcmp(p->name, "softirq"))
                                local_bh_disable();

                        p->critical_section_begin();
                        err = __intel_engine_reset_bh(engine, NULL);
                        p->critical_section_end();

                        if (strcmp(p->name, "softirq"))
                                local_bh_enable();

                        if (err) {
                                pr_err("intel_engine_reset(%s) failed under %s\n",
                                       engine->name, p->name);
                                break;
                        }
                }

                intel_engine_pm_put(engine);
                if (t->func) {
                        tasklet_enable(t);
                        tasklet_hi_schedule(t);
                }
                if (err)
                        break;
        }

        /* As we poke around the guts, do a full reset before continuing. */
        igt_force_reset(gt);

out_unlock:
        igt_global_reset_unlock(gt);
        intel_gt_pm_put(gt, wakeref);

        return err;
}

int intel_reset_live_selftests(struct drm_i915_private *i915)
{
        static const struct i915_subtest tests[] = {
                SUBTEST(igt_global_reset), /* attempt to recover GPU first */
                SUBTEST(igt_reset_device_stolen),
                SUBTEST(igt_reset_engines_stolen),
                SUBTEST(igt_wedged_reset),
                SUBTEST(igt_atomic_reset),
                SUBTEST(igt_atomic_engine_reset),
        };
        struct intel_gt *gt = to_gt(i915);

        if (!intel_has_gpu_reset(gt))
                return 0;

        if (intel_gt_is_wedged(gt))
                return -EIO; /* we're long past hope of a successful reset */

        return intel_gt_live_subtests(tests, gt);
}