/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2018 Intel Corporation
 */

#include <linux/kref.h>
#include <linux/string_helpers.h>

#include <drm/drm_print.h>

#include "gem/i915_gem_pm.h"
#include "gt/intel_gt.h"

#include "i915_selftest.h"

#include "igt_flush_test.h"
#include "lib_sw_fence.h"

struct live_active {
        struct i915_active base;
        struct kref ref;
        bool retired;
};

static void __live_get(struct live_active *active)
{
        kref_get(&active->ref);
}

static void __live_free(struct live_active *active)
{
        i915_active_fini(&active->base);
        kfree(active);
}

static void __live_release(struct kref *ref)
{
        struct live_active *active = container_of(ref, typeof(*active), ref);

        __live_free(active);
}

static void __live_put(struct live_active *active)
{
        kref_put(&active->ref, __live_release);
}

static int __live_active(struct i915_active *base)
{
        struct live_active *active = container_of(base, typeof(*active), base);

        __live_get(active);
        return 0;
}

static void __live_retire(struct i915_active *base)
{
        struct live_active *active = container_of(base, typeof(*active), base);

        active->retired = true;
        __live_put(active);
}

static struct live_active *__live_alloc(struct drm_i915_private *i915)
{
        struct live_active *active;

        active = kzalloc_obj(*active);
        if (!active)
                return NULL;

        kref_init(&active->ref);
        i915_active_init(&active->base, __live_active, __live_retire, 0);

        return active;
}

static struct live_active *
__live_active_setup(struct drm_i915_private *i915)
{
        struct intel_engine_cs *engine;
        struct i915_sw_fence *submit;
        struct live_active *active;
        unsigned int count = 0;
        int err = 0;

        active = __live_alloc(i915);
        if (!active)
                return ERR_PTR(-ENOMEM);

        submit = heap_fence_create(GFP_KERNEL);
        if (!submit) {
                kfree(active);
                return ERR_PTR(-ENOMEM);
        }

        err = i915_active_acquire(&active->base);
        if (err)
                goto out;

        for_each_uabi_engine(engine, i915) {
                struct i915_request *rq;

                rq = intel_engine_create_kernel_request(engine);
                if (IS_ERR(rq)) {
                        err = PTR_ERR(rq);
                        break;
                }

                err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
                                                       submit,
                                                       GFP_KERNEL);
                if (err >= 0)
                        err = i915_active_add_request(&active->base, rq);
                i915_request_add(rq);
                if (err) {
                        pr_err("Failed to track active ref!\n");
                        break;
                }

                count++;
        }

        i915_active_release(&active->base);
        if (READ_ONCE(active->retired) && count) {
                pr_err("i915_active retired before submission!\n");
                err = -EINVAL;
        }
        if (atomic_read(&active->base.count) != count) {
                pr_err("i915_active not tracking all requests, found %d, expected %d\n",
                       atomic_read(&active->base.count), count);
                err = -EINVAL;
        }

out:
        i915_sw_fence_commit(submit);
        heap_fence_put(submit);
        if (err) {
                __live_put(active);
                active = ERR_PTR(err);
        }

        return active;
}

static int live_active_wait(void *arg)
{
        struct drm_i915_private *i915 = arg;
        struct live_active *active;
        int err = 0;

        /* Check that we get a callback when requests retire upon waiting */

        active = __live_active_setup(i915);
        if (IS_ERR(active))
                return PTR_ERR(active);

        __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
        if (!READ_ONCE(active->retired)) {
                struct drm_printer p = drm_err_printer(&i915->drm, __func__);

                drm_printf(&p, "i915_active not retired after waiting!\n");
                i915_active_print(&active->base, &p);

                err = -EINVAL;
        }

        __live_put(active);

        if (igt_flush_test(i915))
                err = -EIO;

        return err;
}

static int live_active_retire(void *arg)
{
        struct drm_i915_private *i915 = arg;
        struct live_active *active;
        int err = 0;

        /* Check that we get a callback when requests are indirectly retired */

        active = __live_active_setup(i915);
        if (IS_ERR(active))
                return PTR_ERR(active);

        /* waits for & retires all requests */
        if (igt_flush_test(i915))
                err = -EIO;

        if (!READ_ONCE(active->retired)) {
                struct drm_printer p = drm_err_printer(&i915->drm, __func__);

                drm_printf(&p, "i915_active not retired after flushing!\n");
                i915_active_print(&active->base, &p);

                err = -EINVAL;
        }

        __live_put(active);

        return err;
}

static int live_active_barrier(void *arg)
{
        struct drm_i915_private *i915 = arg;
        struct intel_engine_cs *engine;
        struct live_active *active;
        int err = 0;

        /* Check that we get a callback when requests retire upon waiting */

        active = __live_alloc(i915);
        if (!active)
                return -ENOMEM;

        err = i915_active_acquire(&active->base);
        if (err)
                goto out;

        for_each_uabi_engine(engine, i915) {
                err = i915_active_acquire_preallocate_barrier(&active->base,
                                                              engine);
                if (err)
                        break;

                i915_active_acquire_barrier(&active->base);
        }

        i915_active_release(&active->base);
        if (err)
                goto out;

        __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
        if (!READ_ONCE(active->retired)) {
                pr_err("i915_active not retired after flushing barriers!\n");
                err = -EINVAL;
        }

out:
        __live_put(active);

        if (igt_flush_test(i915))
                err = -EIO;

        return err;
}

int i915_active_live_selftests(struct drm_i915_private *i915)
{
        static const struct i915_subtest tests[] = {
                SUBTEST(live_active_wait),
                SUBTEST(live_active_retire),
                SUBTEST(live_active_barrier),
        };

        if (intel_gt_is_wedged(to_gt(i915)))
                return 0;

        return i915_subtests(tests, i915);
}

static struct intel_engine_cs *node_to_barrier(struct active_node *it)
{
        struct intel_engine_cs *engine;

        if (!is_barrier(&it->base))
                return NULL;

        engine = __barrier_to_engine(it);
        smp_rmb(); /* serialise with add_active_barriers */
        if (!is_barrier(&it->base))
                return NULL;

        return engine;
}

void i915_active_print(struct i915_active *ref, struct drm_printer *m)
{
        drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire);
        drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count));
        drm_printf(m, "\tpreallocated barriers? %s\n",
                   str_yes_no(!llist_empty(&ref->preallocated_barriers)));

        if (i915_active_acquire_if_busy(ref)) {
                struct active_node *it, *n;

                rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
                        struct intel_engine_cs *engine;

                        engine = node_to_barrier(it);
                        if (engine) {
                                drm_printf(m, "\tbarrier: %s\n", engine->name);
                                continue;
                        }

                        if (i915_active_fence_isset(&it->base)) {
                                drm_printf(m,
                                           "\ttimeline: %llx\n", it->timeline);
                                continue;
                        }
                }

                i915_active_release(ref);
        }
}

static void spin_unlock_wait(spinlock_t *lock)
{
        spin_lock_irq(lock);
        spin_unlock_irq(lock);
}

static void active_flush(struct i915_active *ref,
                         struct i915_active_fence *active)
{
        struct dma_fence *fence;

        fence = xchg(__active_fence_slot(active), NULL);
        if (!fence)
                return;

        spin_lock_irq(fence->lock);
        __list_del_entry(&active->cb.node);
        spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */
        atomic_dec(&ref->count);

        GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
}

void i915_active_unlock_wait(struct i915_active *ref)
{
        if (i915_active_acquire_if_busy(ref)) {
                struct active_node *it, *n;

                /* Wait for all active callbacks */
                rcu_read_lock();
                active_flush(ref, &ref->excl);
                rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node)
                        active_flush(ref, &it->base);
                rcu_read_unlock();

                i915_active_release(ref);
        }

        /* And wait for the retire callback */
        spin_unlock_wait(&ref->tree_lock);

        /* ... which may have been on a thread instead */
        flush_work(&ref->work);
}