/*
 * Copyright 2018 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#ifndef AMDGPU_DOORBELL_H
#define AMDGPU_DOORBELL_H

/*
 * GPU doorbell structures, functions & helpers
 */
struct amdgpu_doorbell {
        /* doorbell mmio */
        resource_size_t         base;
        resource_size_t         size;

        /* Number of doorbells reserved for amdgpu kernel driver */
        u32 num_kernel_doorbells;

        /* Kernel doorbells */
        struct amdgpu_bo *kernel_doorbells;

        /* For CPU access of doorbells */
        uint32_t *cpu_addr;
};

/* Reserved doorbells for amdgpu (including multimedia).
 * KFD can use all the rest in the 2M doorbell bar.
 * For asic before vega10, doorbell is 32-bit, so the
 * index/offset is in dword. For vega10 and after, doorbell
 * can be 64-bit, so the index defined is in qword.
 */
struct amdgpu_doorbell_index {
        uint32_t kiq;
        uint32_t mec_ring0;
        uint32_t mec_ring1;
        uint32_t mec_ring2;
        uint32_t mec_ring3;
        uint32_t mec_ring4;
        uint32_t mec_ring5;
        uint32_t mec_ring6;
        uint32_t mec_ring7;
        uint32_t userqueue_start;
        uint32_t userqueue_end;
        uint32_t gfx_ring0;
        uint32_t gfx_ring1;
        uint32_t gfx_userqueue_start;
        uint32_t gfx_userqueue_end;
        uint32_t sdma_engine[16];
        uint32_t mes_ring0;
        uint32_t mes_ring1;
        uint32_t ih;
        union {
                struct {
                        uint32_t vcn_ring0_1;
                        uint32_t vcn_ring2_3;
                        uint32_t vcn_ring4_5;
                        uint32_t vcn_ring6_7;
                } vcn;
                struct {
                        uint32_t uvd_ring0_1;
                        uint32_t uvd_ring2_3;
                        uint32_t uvd_ring4_5;
                        uint32_t uvd_ring6_7;
                        uint32_t vce_ring0_1;
                        uint32_t vce_ring2_3;
                        uint32_t vce_ring4_5;
                        uint32_t vce_ring6_7;
                } uvd_vce;
        };
        uint32_t vpe_ring;
        uint32_t first_non_cp;
        uint32_t last_non_cp;
        uint32_t max_assignment;
        /* Per engine SDMA doorbell size in dword */
        uint32_t sdma_doorbell_range;
        /* Per xcc doorbell size for KIQ/KCQ */
        uint32_t xcc_doorbell_range;
};

enum AMDGPU_DOORBELL_ASSIGNMENT {
        AMDGPU_DOORBELL_KIQ                     = 0x000,
        AMDGPU_DOORBELL_HIQ                     = 0x001,
        AMDGPU_DOORBELL_DIQ                     = 0x002,
        AMDGPU_DOORBELL_MEC_RING0               = 0x010,
        AMDGPU_DOORBELL_MEC_RING1               = 0x011,
        AMDGPU_DOORBELL_MEC_RING2               = 0x012,
        AMDGPU_DOORBELL_MEC_RING3               = 0x013,
        AMDGPU_DOORBELL_MEC_RING4               = 0x014,
        AMDGPU_DOORBELL_MEC_RING5               = 0x015,
        AMDGPU_DOORBELL_MEC_RING6               = 0x016,
        AMDGPU_DOORBELL_MEC_RING7               = 0x017,
        AMDGPU_DOORBELL_GFX_RING0               = 0x020,
        AMDGPU_DOORBELL_sDMA_ENGINE0            = 0x1E0,
        AMDGPU_DOORBELL_sDMA_ENGINE1            = 0x1E1,
        AMDGPU_DOORBELL_IH                      = 0x1E8,
        AMDGPU_DOORBELL_MAX_ASSIGNMENT          = 0x3FF,
        AMDGPU_DOORBELL_INVALID                 = 0xFFFF
};

enum AMDGPU_VEGA20_DOORBELL_ASSIGNMENT {

        /* Compute + GFX: 0~255 */
        AMDGPU_VEGA20_DOORBELL_KIQ                     = 0x000,
        AMDGPU_VEGA20_DOORBELL_HIQ                     = 0x001,
        AMDGPU_VEGA20_DOORBELL_DIQ                     = 0x002,
        AMDGPU_VEGA20_DOORBELL_MEC_RING0               = 0x003,
        AMDGPU_VEGA20_DOORBELL_MEC_RING1               = 0x004,
        AMDGPU_VEGA20_DOORBELL_MEC_RING2               = 0x005,
        AMDGPU_VEGA20_DOORBELL_MEC_RING3               = 0x006,
        AMDGPU_VEGA20_DOORBELL_MEC_RING4               = 0x007,
        AMDGPU_VEGA20_DOORBELL_MEC_RING5               = 0x008,
        AMDGPU_VEGA20_DOORBELL_MEC_RING6               = 0x009,
        AMDGPU_VEGA20_DOORBELL_MEC_RING7               = 0x00A,
        AMDGPU_VEGA20_DOORBELL_USERQUEUE_START         = 0x00B,
        AMDGPU_VEGA20_DOORBELL_USERQUEUE_END           = 0x08A,
        AMDGPU_VEGA20_DOORBELL_GFX_RING0               = 0x08B,
        /* SDMA:256~335*/
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0            = 0x100,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1            = 0x10A,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2            = 0x114,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3            = 0x11E,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4            = 0x128,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5            = 0x132,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6            = 0x13C,
        AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7            = 0x146,
        /* IH: 376~391 */
        AMDGPU_VEGA20_DOORBELL_IH                      = 0x178,
        /* MMSCH: 392~407
         * overlap the doorbell assignment with VCN as they are  mutually exclusive
         * VCN engine's doorbell is 32 bit and two VCN ring share one QWORD
         */
        AMDGPU_VEGA20_DOORBELL64_VCN0_1                  = 0x188, /* VNC0 */
        AMDGPU_VEGA20_DOORBELL64_VCN2_3                  = 0x189,
        AMDGPU_VEGA20_DOORBELL64_VCN4_5                  = 0x18A,
        AMDGPU_VEGA20_DOORBELL64_VCN6_7                  = 0x18B,

        AMDGPU_VEGA20_DOORBELL64_VCN8_9                  = 0x18C, /* VNC1 */
        AMDGPU_VEGA20_DOORBELL64_VCNa_b                  = 0x18D,
        AMDGPU_VEGA20_DOORBELL64_VCNc_d                  = 0x18E,
        AMDGPU_VEGA20_DOORBELL64_VCNe_f                  = 0x18F,

        AMDGPU_VEGA20_DOORBELL64_UVD_RING0_1             = 0x188,
        AMDGPU_VEGA20_DOORBELL64_UVD_RING2_3             = 0x189,
        AMDGPU_VEGA20_DOORBELL64_UVD_RING4_5             = 0x18A,
        AMDGPU_VEGA20_DOORBELL64_UVD_RING6_7             = 0x18B,

        AMDGPU_VEGA20_DOORBELL64_VCE_RING0_1             = 0x18C,
        AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3             = 0x18D,
        AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5             = 0x18E,
        AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7             = 0x18F,

        AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP            = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0,
        AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP             = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7,

        /* kiq/kcq from second XCD. Max 8 XCDs */
        AMDGPU_VEGA20_DOORBELL_XCC1_KIQ_START             = 0x190,
        /* 8 compute rings per GC. Max to 0x1CE */
        AMDGPU_VEGA20_DOORBELL_XCC1_MEC_RING0_START       = 0x197,

        /* AID1 SDMA: 0x1D0 ~ 0x1F7 */
        AMDGPU_VEGA20_DOORBELL_AID1_sDMA_START           = 0x1D0,

        AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT            = 0x1F7,
        AMDGPU_VEGA20_DOORBELL_INVALID                   = 0xFFFF
};

enum AMDGPU_NAVI10_DOORBELL_ASSIGNMENT {

        /* Compute + GFX: 0~255 */
        AMDGPU_NAVI10_DOORBELL_KIQ                      = 0x000,
        AMDGPU_NAVI10_DOORBELL_HIQ                      = 0x001,
        AMDGPU_NAVI10_DOORBELL_DIQ                      = 0x002,
        AMDGPU_NAVI10_DOORBELL_MEC_RING0                = 0x003,
        AMDGPU_NAVI10_DOORBELL_MEC_RING1                = 0x004,
        AMDGPU_NAVI10_DOORBELL_MEC_RING2                = 0x005,
        AMDGPU_NAVI10_DOORBELL_MEC_RING3                = 0x006,
        AMDGPU_NAVI10_DOORBELL_MEC_RING4                = 0x007,
        AMDGPU_NAVI10_DOORBELL_MEC_RING5                = 0x008,
        AMDGPU_NAVI10_DOORBELL_MEC_RING6                = 0x009,
        AMDGPU_NAVI10_DOORBELL_MEC_RING7                = 0x00A,
        AMDGPU_NAVI10_DOORBELL_MES_RING0                = 0x00B,
        AMDGPU_NAVI10_DOORBELL_MES_RING1                = 0x00C,
        AMDGPU_NAVI10_DOORBELL_USERQUEUE_START          = 0x00D,
        AMDGPU_NAVI10_DOORBELL_USERQUEUE_END            = 0x08A,
        AMDGPU_NAVI10_DOORBELL_GFX_RING0                = 0x08B,
        AMDGPU_NAVI10_DOORBELL_GFX_RING1                = 0x08C,
        AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_START      = 0x08D,
        AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_END        = 0x0FF,

        /* SDMA:256~335*/
        AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0             = 0x100,
        AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1             = 0x10A,
        AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE2             = 0x114,
        AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE3             = 0x11E,
        /* IH: 376~391 */
        AMDGPU_NAVI10_DOORBELL_IH                       = 0x178,
        /* MMSCH: 392~407
         * overlap the doorbell assignment with VCN as they are  mutually exclusive
         * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
         */
        AMDGPU_NAVI10_DOORBELL64_VCN0_1                 = 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
        AMDGPU_NAVI10_DOORBELL64_VCN2_3                 = 0x189,
        AMDGPU_NAVI10_DOORBELL64_VCN4_5                 = 0x18A,
        AMDGPU_NAVI10_DOORBELL64_VCN6_7                 = 0x18B,

        AMDGPU_NAVI10_DOORBELL64_VCN8_9                 = 0x18C,
        AMDGPU_NAVI10_DOORBELL64_VCNa_b                 = 0x18D,
        AMDGPU_NAVI10_DOORBELL64_VCNc_d                 = 0x18E,
        AMDGPU_NAVI10_DOORBELL64_VCNe_f                 = 0x18F,

        AMDGPU_NAVI10_DOORBELL64_VPE                    = 0x190,

        AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP           = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0,
        AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP            = AMDGPU_NAVI10_DOORBELL64_VPE,

        AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT           = AMDGPU_NAVI10_DOORBELL64_VPE,
        AMDGPU_NAVI10_DOORBELL_INVALID                  = 0xFFFF
};

/*
 * 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space
 */
enum AMDGPU_DOORBELL64_ASSIGNMENT {
        /*
         * All compute related doorbells: kiq, hiq, diq, traditional compute queue, user queue, should locate in
         * a continues range so that programming CP_MEC_DOORBELL_RANGE_LOWER/UPPER can cover this range.
         *  Compute related doorbells are allocated from 0x00 to 0x8a
         */


        /* kernel scheduling */
        AMDGPU_DOORBELL64_KIQ                     = 0x00,

        /* HSA interface queue and debug queue */
        AMDGPU_DOORBELL64_HIQ                     = 0x01,
        AMDGPU_DOORBELL64_DIQ                     = 0x02,

        /* Compute engines */
        AMDGPU_DOORBELL64_MEC_RING0               = 0x03,
        AMDGPU_DOORBELL64_MEC_RING1               = 0x04,
        AMDGPU_DOORBELL64_MEC_RING2               = 0x05,
        AMDGPU_DOORBELL64_MEC_RING3               = 0x06,
        AMDGPU_DOORBELL64_MEC_RING4               = 0x07,
        AMDGPU_DOORBELL64_MEC_RING5               = 0x08,
        AMDGPU_DOORBELL64_MEC_RING6               = 0x09,
        AMDGPU_DOORBELL64_MEC_RING7               = 0x0a,

        /* User queue doorbell range (128 doorbells) */
        AMDGPU_DOORBELL64_USERQUEUE_START         = 0x0b,
        AMDGPU_DOORBELL64_USERQUEUE_END           = 0x8a,

        /* Graphics engine */
        AMDGPU_DOORBELL64_GFX_RING0               = 0x8b,

        /*
         * Other graphics doorbells can be allocated here: from 0x8c to 0xdf
         * Graphics voltage island aperture 1
         * default non-graphics QWORD index is 0xe0 - 0xFF inclusive
         */

        /* For vega10 sriov, the sdma doorbell must be fixed as follow
         * to keep the same setting with host driver, or it will
         * happen conflicts
         */
        AMDGPU_DOORBELL64_sDMA_ENGINE0            = 0xF0,
        AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE0     = 0xF1,
        AMDGPU_DOORBELL64_sDMA_ENGINE1            = 0xF2,
        AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE1     = 0xF3,

        /* Interrupt handler */
        AMDGPU_DOORBELL64_IH                      = 0xF4,  /* For legacy interrupt ring buffer */
        AMDGPU_DOORBELL64_IH_RING1                = 0xF5,  /* For page migration request log */
        AMDGPU_DOORBELL64_IH_RING2                = 0xF6,  /* For page migration translation/invalidation log */

        /* VCN engine use 32 bits doorbell  */
        AMDGPU_DOORBELL64_VCN0_1                  = 0xF8, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
        AMDGPU_DOORBELL64_VCN2_3                  = 0xF9,
        AMDGPU_DOORBELL64_VCN4_5                  = 0xFA,
        AMDGPU_DOORBELL64_VCN6_7                  = 0xFB,

        /* overlap the doorbell assignment with VCN as they are  mutually exclusive
         * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
         */
        AMDGPU_DOORBELL64_UVD_RING0_1             = 0xF8,
        AMDGPU_DOORBELL64_UVD_RING2_3             = 0xF9,
        AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFA,
        AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFB,

        AMDGPU_DOORBELL64_VCE_RING0_1             = 0xFC,
        AMDGPU_DOORBELL64_VCE_RING2_3             = 0xFD,
        AMDGPU_DOORBELL64_VCE_RING4_5             = 0xFE,
        AMDGPU_DOORBELL64_VCE_RING6_7             = 0xFF,

        AMDGPU_DOORBELL64_FIRST_NON_CP            = AMDGPU_DOORBELL64_sDMA_ENGINE0,
        AMDGPU_DOORBELL64_LAST_NON_CP             = AMDGPU_DOORBELL64_VCE_RING6_7,

        AMDGPU_DOORBELL64_MAX_ASSIGNMENT          = 0xFF,
        AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
};

enum AMDGPU_DOORBELL_ASSIGNMENT_LAYOUT1 {

        /* XCC0: 0x00 ~20, XCC1: 20 ~ 2F ... */

        /* KIQ/HIQ/DIQ */
        AMDGPU_DOORBELL_LAYOUT1_KIQ_START               = 0x000,
        AMDGPU_DOORBELL_LAYOUT1_HIQ                     = 0x001,
        AMDGPU_DOORBELL_LAYOUT1_DIQ                     = 0x002,
        /* Compute: 0x08 ~ 0x20  */
        AMDGPU_DOORBELL_LAYOUT1_MEC_RING_START          = 0x008,
        AMDGPU_DOORBELL_LAYOUT1_MEC_RING_END            = 0x00F,
        AMDGPU_DOORBELL_LAYOUT1_USERQUEUE_START         = 0x010,
        AMDGPU_DOORBELL_LAYOUT1_USERQUEUE_END           = 0x01F,
        AMDGPU_DOORBELL_LAYOUT1_XCC_RANGE               = 0x020,

        /* SDMA: 0x100 ~ 0x19F */
        AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_START       = 0x100,
        AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_END         = 0x19F,
        /* IH: 0x1A0 ~ 0x1AF */
        AMDGPU_DOORBELL_LAYOUT1_IH                      = 0x1A0,
        /* VCN: 0x1B0 ~ 0x1E8 */
        AMDGPU_DOORBELL_LAYOUT1_VCN_START               = 0x1B0,
        AMDGPU_DOORBELL_LAYOUT1_VCN_END                 = 0x1E8,

        AMDGPU_DOORBELL_LAYOUT1_FIRST_NON_CP            = AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_START,
        AMDGPU_DOORBELL_LAYOUT1_LAST_NON_CP             = AMDGPU_DOORBELL_LAYOUT1_VCN_END,

        AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT          = 0x1E8,
        AMDGPU_DOORBELL_LAYOUT1_INVALID                 = 0xFFFF
};

u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);

/*
 * GPU doorbell aperture helpers function.
 */
int amdgpu_doorbell_init(struct amdgpu_device *adev);
void amdgpu_doorbell_fini(struct amdgpu_device *adev);
int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev);
uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
                                      struct amdgpu_bo *db_bo,
                                      uint32_t doorbell_index,
                                      uint32_t db_size);

#define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
#define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
#define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
#define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v))

#endif