/* SPDX-License-Identifier: MIT */
/*
 * Copyright © 2019 Intel Corporation
 */

#ifndef INTEL_RING_H
#define INTEL_RING_H

#include "i915_gem.h" /* GEM_BUG_ON */
#include "i915_request.h"
#include "intel_ring_types.h"

struct intel_engine_cs;

struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs *engine, int size);

u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords);

unsigned int intel_ring_update_space(struct intel_ring *ring);

void __intel_ring_pin(struct intel_ring *ring);
int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww);
void intel_ring_unpin(struct intel_ring *ring);
void intel_ring_reset(struct intel_ring *ring, u32 tail);

void intel_ring_free(struct kref *ref);

static inline struct intel_ring *intel_ring_get(struct intel_ring *ring)
{
        kref_get(&ring->ref);
        return ring;
}

static inline void intel_ring_put(struct intel_ring *ring)
{
        kref_put(&ring->ref, intel_ring_free);
}

static inline void intel_ring_advance(struct i915_request *rq, u32 *cs)
{
        /* Dummy function.
         *
         * This serves as a placeholder in the code so that the reader
         * can compare against the preceding intel_ring_begin() and
         * check that the number of dwords emitted matches the space
         * reserved for the command packet (i.e. the value passed to
         * intel_ring_begin()).
         */
        GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs);
        GEM_BUG_ON(!IS_ALIGNED(rq->ring->emit, 8)); /* RING_TAIL qword align */
}

static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos)
{
        return pos & (ring->size - 1);
}

static inline int intel_ring_direction(const struct intel_ring *ring,
                                       u32 next, u32 prev)
{
        typecheck(typeof(ring->size), next);
        typecheck(typeof(ring->size), prev);
        return (next - prev) << ring->wrap;
}

static inline bool
intel_ring_offset_valid(const struct intel_ring *ring,
                        unsigned int pos)
{
        if (pos & -ring->size) /* must be strictly within the ring */
                return false;

        if (!IS_ALIGNED(pos, 8)) /* must be qword aligned */
                return false;

        return true;
}

static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr)
{
        /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
        u32 offset = addr - rq->ring->vaddr;

        GEM_BUG_ON(offset > rq->ring->size);
        return intel_ring_wrap(rq->ring, offset);
}

static inline void
assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
{
        unsigned int head = READ_ONCE(ring->head);

        GEM_BUG_ON(!intel_ring_offset_valid(ring, tail));

        /*
         * "Ring Buffer Use"
         *      Gen2 BSpec "1. Programming Environment" / 1.4.4.6
         *      Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
         *      Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
         * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
         * same cacheline, the Head Pointer must not be greater than the Tail
         * Pointer."
         *
         * We use ring->head as the last known location of the actual RING_HEAD,
         * it may have advanced but in the worst case it is equally the same
         * as ring->head and so we should never program RING_TAIL to advance
         * into the same cacheline as ring->head.
         */
#define cacheline(a) round_down(a, CACHELINE_BYTES)
        GEM_BUG_ON(cacheline(tail) == cacheline(head) && tail < head);
#undef cacheline
}

static inline unsigned int
intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
{
        /* Whilst writes to the tail are strictly order, there is no
         * serialisation between readers and the writers. The tail may be
         * read by i915_request_retire() just as it is being updated
         * by execlists, as although the breadcrumb is complete, the context
         * switch hasn't been seen.
         */
        assert_ring_tail_valid(ring, tail);
        ring->tail = tail;
        return tail;
}

static inline unsigned int
__intel_ring_space(unsigned int head, unsigned int tail, unsigned int size)
{
        /*
         * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
         * same cacheline, the Head Pointer must not be greater than the Tail
         * Pointer."
         */
        GEM_BUG_ON(!is_power_of_2(size));
        return (head - tail - CACHELINE_BYTES) & (size - 1);
}

#endif /* INTEL_RING_H */