root/drivers/net/wireless/ath/ath12k/hal.c 
// SPDX-License-Identifier: BSD-3-Clause-Clear
/*
 * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
 */
#include <linux/dma-mapping.h>
#include "debug.h"
#include "hif.h"

static void ath12k_hal_ce_dst_setup(struct ath12k_base *ab,
                                    struct hal_srng *srng, int ring_num)
{
        ab->hal.ops->ce_dst_setup(ab, srng, ring_num);
}

static void ath12k_hal_srng_src_hw_init(struct ath12k_base *ab,
                                        struct hal_srng *srng)
{
        ab->hal.ops->srng_src_hw_init(ab, srng);
}

static void ath12k_hal_srng_dst_hw_init(struct ath12k_base *ab,
                                        struct hal_srng *srng)
{
        ab->hal.ops->srng_dst_hw_init(ab, srng);
}

static void ath12k_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab,
                                              struct hal_srng *srng)
{
        ab->hal.ops->set_umac_srng_ptr_addr(ab, srng);
}

static int ath12k_hal_srng_get_ring_id(struct ath12k_hal *hal,
                                       enum hal_ring_type type,
                                       int ring_num, int mac_id)
{
        return hal->ops->srng_get_ring_id(hal, type, ring_num, mac_id);
}

int ath12k_hal_srng_update_shadow_config(struct ath12k_base *ab,
                                         enum hal_ring_type ring_type,
                                         int ring_num)
{
        return ab->hal.ops->srng_update_shadow_config(ab, ring_type,
                                                          ring_num);
}

u32 ath12k_hal_ce_get_desc_size(struct ath12k_hal *hal, enum hal_ce_desc type)
{
        return hal->ops->ce_get_desc_size(type);
}

void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id)
{
        ab->hal.ops->tx_set_dscp_tid_map(ab, id);
}

void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab,
                                           u32 bank_config, u8 bank_id)
{
        ab->hal.ops->tx_configure_bank_register(ab, bank_config, bank_id);
}

void ath12k_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab)
{
        ab->hal.ops->reoq_lut_addr_read_enable(ab);
}

void ath12k_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab)
{
        ab->hal.ops->reoq_lut_set_max_peerid(ab);
}

void ath12k_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr)
{
        ab->hal.ops->write_ml_reoq_lut_addr(ab, paddr);
}

void ath12k_hal_write_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr)
{
        ab->hal.ops->write_reoq_lut_addr(ab, paddr);
}

void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab,
                                     struct hal_wbm_idle_scatter_list *sbuf,
                                     u32 nsbufs, u32 tot_link_desc,
                                     u32 end_offset)
{
        ab->hal.ops->setup_link_idle_list(ab, sbuf, nsbufs, tot_link_desc,
                                              end_offset);
}

void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map)
{
        ab->hal.ops->reo_hw_setup(ab, ring_hash_map);
}

void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, struct hal_srng *srng)
{
        ab->hal.ops->reo_init_cmd_ring(ab, srng);
}

void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab)
{
        ab->hal.ops->reo_shared_qaddr_cache_clear(ab);
}
EXPORT_SYMBOL(ath12k_hal_reo_shared_qaddr_cache_clear);

void ath12k_hal_rx_buf_addr_info_set(struct ath12k_hal *hal,
                                     struct ath12k_buffer_addr *binfo,
                                     dma_addr_t paddr, u32 cookie, u8 manager)
{
        hal->ops->rx_buf_addr_info_set(binfo, paddr, cookie, manager);
}

void ath12k_hal_rx_buf_addr_info_get(struct ath12k_hal *hal,
                                     struct ath12k_buffer_addr *binfo,
                                     dma_addr_t *paddr, u32 *msdu_cookies,
                                     u8 *rbm)
{
        hal->ops->rx_buf_addr_info_get(binfo, paddr, msdu_cookies, rbm);
}

void ath12k_hal_rx_msdu_list_get(struct ath12k_hal *hal, struct ath12k *ar,
                                 void *link_desc,
                                 void *msdu_list,
                                 u16 *num_msdus)
{
        hal->ops->rx_msdu_list_get(ar, link_desc, msdu_list, num_msdus);
}

void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc,
                                         dma_addr_t *paddr,
                                         u32 *sw_cookie,
                                         struct ath12k_buffer_addr **pp_buf_addr,
                                         u8 *rbm, u32 *msdu_cnt)
{
        hal->ops->rx_reo_ent_buf_paddr_get(rx_desc, paddr, sw_cookie,
                                           pp_buf_addr, rbm, msdu_cnt);
}

void ath12k_hal_cc_config(struct ath12k_base *ab)
{
        ab->hal.ops->cc_config(ab);
}

enum hal_rx_buf_return_buf_manager
ath12k_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id)
{
        return hal->ops->get_idle_link_rbm(hal, device_id);
}

static int ath12k_hal_alloc_cont_rdp(struct ath12k_hal *hal)
{
        size_t size;

        size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
        hal->rdp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->rdp.paddr,
                                            GFP_KERNEL);
        if (!hal->rdp.vaddr)
                return -ENOMEM;

        return 0;
}

static void ath12k_hal_free_cont_rdp(struct ath12k_hal *hal)
{
        size_t size;

        if (!hal->rdp.vaddr)
                return;

        size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
        dma_free_coherent(hal->dev, size,
                          hal->rdp.vaddr, hal->rdp.paddr);
        hal->rdp.vaddr = NULL;
}

static int ath12k_hal_alloc_cont_wrp(struct ath12k_hal *hal)
{
        size_t size;

        size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS);
        hal->wrp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->wrp.paddr,
                                            GFP_KERNEL);
        if (!hal->wrp.vaddr)
                return -ENOMEM;

        return 0;
}

static void ath12k_hal_free_cont_wrp(struct ath12k_hal *hal)
{
        size_t size;

        if (!hal->wrp.vaddr)
                return;

        size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS);
        dma_free_coherent(hal->dev, size,
                          hal->wrp.vaddr, hal->wrp.paddr);
        hal->wrp.vaddr = NULL;
}

static void ath12k_hal_srng_hw_init(struct ath12k_base *ab,
                                    struct hal_srng *srng)
{
        if (srng->ring_dir == HAL_SRNG_DIR_SRC)
                ath12k_hal_srng_src_hw_init(ab, srng);
        else
                ath12k_hal_srng_dst_hw_init(ab, srng);
}

int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type)
{
        struct hal_srng_config *srng_config;

        if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
                return -EINVAL;

        srng_config = &ab->hal.srng_config[ring_type];

        return (srng_config->entry_size << 2);
}
EXPORT_SYMBOL(ath12k_hal_srng_get_entrysize);

int ath12k_hal_srng_get_max_entries(struct ath12k_base *ab, u32 ring_type)
{
        struct hal_srng_config *srng_config;

        if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
                return -EINVAL;

        srng_config = &ab->hal.srng_config[ring_type];

        return (srng_config->max_size / srng_config->entry_size);
}

void ath12k_hal_srng_get_params(struct ath12k_base *ab, struct hal_srng *srng,
                                struct hal_srng_params *params)
{
        params->ring_base_paddr = srng->ring_base_paddr;
        params->ring_base_vaddr = srng->ring_base_vaddr;
        params->num_entries = srng->num_entries;
        params->intr_timer_thres_us = srng->intr_timer_thres_us;
        params->intr_batch_cntr_thres_entries =
                srng->intr_batch_cntr_thres_entries;
        params->low_threshold = srng->u.src_ring.low_threshold;
        params->msi_addr = srng->msi_addr;
        params->msi2_addr = srng->msi2_addr;
        params->msi_data = srng->msi_data;
        params->msi2_data = srng->msi2_data;
        params->flags = srng->flags;
}
EXPORT_SYMBOL(ath12k_hal_srng_get_params);

dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab,
                                       struct hal_srng *srng)
{
        if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
                return 0;

        if (srng->ring_dir == HAL_SRNG_DIR_SRC)
                return ab->hal.wrp.paddr +
                       ((unsigned long)srng->u.src_ring.hp_addr -
                        (unsigned long)ab->hal.wrp.vaddr);
        else
                return ab->hal.rdp.paddr +
                       ((unsigned long)srng->u.dst_ring.hp_addr -
                         (unsigned long)ab->hal.rdp.vaddr);
}

dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab,
                                       struct hal_srng *srng)
{
        if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
                return 0;

        if (srng->ring_dir == HAL_SRNG_DIR_SRC)
                return ab->hal.rdp.paddr +
                       ((unsigned long)srng->u.src_ring.tp_addr -
                        (unsigned long)ab->hal.rdp.vaddr);
        else
                return ab->hal.wrp.paddr +
                       ((unsigned long)srng->u.dst_ring.tp_addr -
                        (unsigned long)ab->hal.wrp.vaddr);
}

void ath12k_hal_ce_src_set_desc(struct ath12k_hal *hal,
                                struct hal_ce_srng_src_desc *desc,
                                dma_addr_t paddr, u32 len, u32 id,
                                u8 byte_swap_data)
{
        hal->ops->ce_src_set_desc(desc, paddr, len, id, byte_swap_data);
}

void ath12k_hal_ce_dst_set_desc(struct ath12k_hal *hal,
                                struct hal_ce_srng_dest_desc *desc,
                                dma_addr_t paddr)
{
        hal->ops->ce_dst_set_desc(desc, paddr);
}

u32 ath12k_hal_ce_dst_status_get_length(struct ath12k_hal *hal,
                                        struct hal_ce_srng_dst_status_desc *desc)
{
        return hal->ops->ce_dst_status_get_length(desc);
}

void ath12k_hal_set_link_desc_addr(struct ath12k_hal *hal,
                                   struct hal_wbm_link_desc *desc, u32 cookie,
                                   dma_addr_t paddr, int rbm)
{
        hal->ops->set_link_desc_addr(desc, cookie, paddr, rbm);
}

void *ath12k_hal_srng_dst_peek(struct ath12k_base *ab, struct hal_srng *srng)
{
        lockdep_assert_held(&srng->lock);

        if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp)
                return (srng->ring_base_vaddr + srng->u.dst_ring.tp);

        return NULL;
}
EXPORT_SYMBOL(ath12k_hal_srng_dst_peek);

void *ath12k_hal_srng_dst_get_next_entry(struct ath12k_base *ab,
                                         struct hal_srng *srng)
{
        void *desc;

        lockdep_assert_held(&srng->lock);

        if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp)
                return NULL;

        desc = srng->ring_base_vaddr + srng->u.dst_ring.tp;

        srng->u.dst_ring.tp = (srng->u.dst_ring.tp + srng->entry_size) %
                              srng->ring_size;

        return desc;
}
EXPORT_SYMBOL(ath12k_hal_srng_dst_get_next_entry);

int ath12k_hal_srng_dst_num_free(struct ath12k_base *ab, struct hal_srng *srng,
                                 bool sync_hw_ptr)
{
        u32 tp, hp;

        lockdep_assert_held(&srng->lock);

        tp = srng->u.dst_ring.tp;

        if (sync_hw_ptr) {
                hp = *srng->u.dst_ring.hp_addr;
                srng->u.dst_ring.cached_hp = hp;
        } else {
                hp = srng->u.dst_ring.cached_hp;
        }

        if (hp >= tp)
                return (hp - tp) / srng->entry_size;
        else
                return (srng->ring_size - tp + hp) / srng->entry_size;
}
EXPORT_SYMBOL(ath12k_hal_srng_dst_num_free);

/* Returns number of available entries in src ring */
int ath12k_hal_srng_src_num_free(struct ath12k_base *ab, struct hal_srng *srng,
                                 bool sync_hw_ptr)
{
        u32 tp, hp;

        lockdep_assert_held(&srng->lock);

        hp = srng->u.src_ring.hp;

        if (sync_hw_ptr) {
                tp = *srng->u.src_ring.tp_addr;
                srng->u.src_ring.cached_tp = tp;
        } else {
                tp = srng->u.src_ring.cached_tp;
        }

        if (tp > hp)
                return ((tp - hp) / srng->entry_size) - 1;
        else
                return ((srng->ring_size - hp + tp) / srng->entry_size) - 1;
}

void *ath12k_hal_srng_src_next_peek(struct ath12k_base *ab,
                                    struct hal_srng *srng)
{
        void *desc;
        u32 next_hp;

        lockdep_assert_held(&srng->lock);

        next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;

        if (next_hp == srng->u.src_ring.cached_tp)
                return NULL;

        desc = srng->ring_base_vaddr + next_hp;

        return desc;
}
EXPORT_SYMBOL(ath12k_hal_srng_src_next_peek);

void *ath12k_hal_srng_src_get_next_entry(struct ath12k_base *ab,
                                         struct hal_srng *srng)
{
        void *desc;
        u32 next_hp;

        lockdep_assert_held(&srng->lock);

        /* TODO: Using % is expensive, but we have to do this since size of some
         * SRNG rings is not power of 2 (due to descriptor sizes). Need to see
         * if separate function is defined for rings having power of 2 ring size
         * (TCL2SW, REO2SW, SW2RXDMA and CE rings) so that we can avoid the
         * overhead of % by using mask (with &).
         */
        next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;

        if (next_hp == srng->u.src_ring.cached_tp)
                return NULL;

        desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
        srng->u.src_ring.hp = next_hp;

        /* TODO: Reap functionality is not used by all rings. If particular
         * ring does not use reap functionality, we need not update reap_hp
         * with next_hp pointer. Need to make sure a separate function is used
         * before doing any optimization by removing below code updating
         * reap_hp.
         */
        srng->u.src_ring.reap_hp = next_hp;

        return desc;
}
EXPORT_SYMBOL(ath12k_hal_srng_src_get_next_entry);

void *ath12k_hal_srng_src_peek(struct ath12k_base *ab, struct hal_srng *srng)
{
        lockdep_assert_held(&srng->lock);

        if (((srng->u.src_ring.hp + srng->entry_size) % srng->ring_size) ==
            srng->u.src_ring.cached_tp)
                return NULL;

        return srng->ring_base_vaddr + srng->u.src_ring.hp;
}
EXPORT_SYMBOL(ath12k_hal_srng_src_peek);

void *ath12k_hal_srng_src_reap_next(struct ath12k_base *ab,
                                    struct hal_srng *srng)
{
        void *desc;
        u32 next_reap_hp;

        lockdep_assert_held(&srng->lock);

        next_reap_hp = (srng->u.src_ring.reap_hp + srng->entry_size) %
                       srng->ring_size;

        if (next_reap_hp == srng->u.src_ring.cached_tp)
                return NULL;

        desc = srng->ring_base_vaddr + next_reap_hp;
        srng->u.src_ring.reap_hp = next_reap_hp;

        return desc;
}

void *ath12k_hal_srng_src_get_next_reaped(struct ath12k_base *ab,
                                          struct hal_srng *srng)
{
        void *desc;

        lockdep_assert_held(&srng->lock);

        if (srng->u.src_ring.hp == srng->u.src_ring.reap_hp)
                return NULL;

        desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
        srng->u.src_ring.hp = (srng->u.src_ring.hp + srng->entry_size) %
                              srng->ring_size;

        return desc;
}

void ath12k_hal_srng_access_begin(struct ath12k_base *ab, struct hal_srng *srng)
{
        u32 hp;

        lockdep_assert_held(&srng->lock);

        if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
                srng->u.src_ring.cached_tp =
                        *(volatile u32 *)srng->u.src_ring.tp_addr;
        } else {
                hp = READ_ONCE(*srng->u.dst_ring.hp_addr);

                if (hp != srng->u.dst_ring.cached_hp) {
                        srng->u.dst_ring.cached_hp = hp;
                        /* Make sure descriptor is read after the head
                         * pointer.
                         */
                        dma_rmb();
                }
        }
}
EXPORT_SYMBOL(ath12k_hal_srng_access_begin);

/* Update cached ring head/tail pointers to HW. ath12k_hal_srng_access_begin()
 * should have been called before this.
 */
void ath12k_hal_srng_access_end(struct ath12k_base *ab, struct hal_srng *srng)
{
        lockdep_assert_held(&srng->lock);

        if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) {
                /* For LMAC rings, ring pointer updates are done through FW and
                 * hence written to a shared memory location that is read by FW
                 */
                if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
                        srng->u.src_ring.last_tp =
                                *(volatile u32 *)srng->u.src_ring.tp_addr;
                        /* Make sure descriptor is written before updating the
                         * head pointer.
                         */
                        dma_wmb();
                        WRITE_ONCE(*srng->u.src_ring.hp_addr, srng->u.src_ring.hp);
                } else {
                        srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
                        /* Make sure descriptor is read before updating the
                         * tail pointer.
                         */
                        dma_mb();
                        WRITE_ONCE(*srng->u.dst_ring.tp_addr, srng->u.dst_ring.tp);
                }
        } else {
                if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
                        srng->u.src_ring.last_tp =
                                *(volatile u32 *)srng->u.src_ring.tp_addr;
                        /* Assume implementation use an MMIO write accessor
                         * which has the required wmb() so that the descriptor
                         * is written before the updating the head pointer.
                         */
                        ath12k_hif_write32(ab,
                                           (unsigned long)srng->u.src_ring.hp_addr -
                                           (unsigned long)ab->mem,
                                           srng->u.src_ring.hp);
                } else {
                        srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
                        /* Make sure descriptor is read before updating the
                         * tail pointer.
                         */
                        mb();
                        ath12k_hif_write32(ab,
                                           (unsigned long)srng->u.dst_ring.tp_addr -
                                           (unsigned long)ab->mem,
                                           srng->u.dst_ring.tp);
                }
        }

        srng->timestamp = jiffies;
}
EXPORT_SYMBOL(ath12k_hal_srng_access_end);

int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
                          int ring_num, int mac_id,
                          struct hal_srng_params *params)
{
        struct ath12k_hal *hal = &ab->hal;
        struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
        struct hal_srng *srng;
        int ring_id;
        u32 idx;
        int i;

        ring_id = ath12k_hal_srng_get_ring_id(hal, type, ring_num, mac_id);
        if (ring_id < 0)
                return ring_id;

        srng = &hal->srng_list[ring_id];

        srng->ring_id = ring_id;
        srng->ring_dir = srng_config->ring_dir;
        srng->ring_base_paddr = params->ring_base_paddr;
        srng->ring_base_vaddr = params->ring_base_vaddr;
        srng->entry_size = srng_config->entry_size;
        srng->num_entries = params->num_entries;
        srng->ring_size = srng->entry_size * srng->num_entries;
        srng->intr_batch_cntr_thres_entries =
                                params->intr_batch_cntr_thres_entries;
        srng->intr_timer_thres_us = params->intr_timer_thres_us;
        srng->flags = params->flags;
        srng->msi_addr = params->msi_addr;
        srng->msi2_addr = params->msi2_addr;
        srng->msi_data = params->msi_data;
        srng->msi2_data = params->msi2_data;
        srng->initialized = 1;
        spin_lock_init(&srng->lock);
        lockdep_set_class(&srng->lock, &srng->lock_key);

        for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) {
                srng->hwreg_base[i] = srng_config->reg_start[i] +
                                      (ring_num * srng_config->reg_size[i]);
        }

        memset(srng->ring_base_vaddr, 0,
               (srng->entry_size * srng->num_entries) << 2);

        if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
                srng->u.src_ring.hp = 0;
                srng->u.src_ring.cached_tp = 0;
                srng->u.src_ring.reap_hp = srng->ring_size - srng->entry_size;
                srng->u.src_ring.tp_addr = (void *)(hal->rdp.vaddr + ring_id);
                srng->u.src_ring.low_threshold = params->low_threshold *
                                                 srng->entry_size;
                if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) {
                        ath12k_hal_set_umac_srng_ptr_addr(ab, srng);
                } else {
                        idx = ring_id - HAL_SRNG_RING_ID_DMAC_CMN_ID_START;
                        srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr +
                                                   idx);
                        srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
                }
        } else {
                /* During initialization loop count in all the descriptors
                 * will be set to zero, and HW will set it to 1 on completing
                 * descriptor update in first loop, and increments it by 1 on
                 * subsequent loops (loop count wraps around after reaching
                 * 0xffff). The 'loop_cnt' in SW ring state is the expected
                 * loop count in descriptors updated by HW (to be processed
                 * by SW).
                 */
                srng->u.dst_ring.loop_cnt = 1;
                srng->u.dst_ring.tp = 0;
                srng->u.dst_ring.cached_hp = 0;
                srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id);
                if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) {
                        ath12k_hal_set_umac_srng_ptr_addr(ab, srng);
                } else {
                        /* For PMAC & DMAC rings, tail pointer updates will be done
                         * through FW by writing to a shared memory location
                         */
                        idx = ring_id - HAL_SRNG_RING_ID_DMAC_CMN_ID_START;
                        srng->u.dst_ring.tp_addr = (void *)(hal->wrp.vaddr +
                                                   idx);
                        srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
                }
        }

        if (srng_config->mac_type != ATH12K_HAL_SRNG_UMAC)
                return ring_id;

        ath12k_hal_srng_hw_init(ab, srng);

        if (type == HAL_CE_DST) {
                srng->u.dst_ring.max_buffer_length = params->max_buffer_len;
                ath12k_hal_ce_dst_setup(ab, srng, ring_num);
        }

        return ring_id;
}

void ath12k_hal_srng_shadow_config(struct ath12k_base *ab)
{
        struct ath12k_hal *hal = &ab->hal;
        int ring_type, ring_num;

        /* update all the non-CE srngs. */
        for (ring_type = 0; ring_type < HAL_MAX_RING_TYPES; ring_type++) {
                struct hal_srng_config *srng_config = &hal->srng_config[ring_type];

                if (ring_type == HAL_CE_SRC ||
                    ring_type == HAL_CE_DST ||
                        ring_type == HAL_CE_DST_STATUS)
                        continue;

                if (srng_config->mac_type == ATH12K_HAL_SRNG_DMAC ||
                    srng_config->mac_type == ATH12K_HAL_SRNG_PMAC)
                        continue;

                for (ring_num = 0; ring_num < srng_config->max_rings; ring_num++)
                        ath12k_hal_srng_update_shadow_config(ab, ring_type, ring_num);
        }
}

void ath12k_hal_srng_get_shadow_config(struct ath12k_base *ab,
                                       u32 **cfg, u32 *len)
{
        struct ath12k_hal *hal = &ab->hal;

        *len = hal->num_shadow_reg_configured;
        *cfg = hal->shadow_reg_addr;
}

void ath12k_hal_srng_shadow_update_hp_tp(struct ath12k_base *ab,
                                         struct hal_srng *srng)
{
        lockdep_assert_held(&srng->lock);

        /* check whether the ring is empty. Update the shadow
         * HP only when then ring isn't' empty.
         */
        if (srng->ring_dir == HAL_SRNG_DIR_SRC &&
            *srng->u.src_ring.tp_addr != srng->u.src_ring.hp)
                ath12k_hal_srng_access_end(ab, srng);
}

static void ath12k_hal_register_srng_lock_keys(struct ath12k_hal *hal)
{
        u32 ring_id;

        for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
                lockdep_register_key(&hal->srng_list[ring_id].lock_key);
}

static void ath12k_hal_unregister_srng_lock_keys(struct ath12k_hal *hal)
{
        u32 ring_id;

        for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
                lockdep_unregister_key(&hal->srng_list[ring_id].lock_key);
}

int ath12k_hal_srng_init(struct ath12k_base *ab)
{
        struct ath12k_hal *hal = &ab->hal;
        int ret;

        ret = hal->ops->create_srng_config(hal);
        if (ret)
                goto err_hal;

        hal->dev = ab->dev;

        ret = ath12k_hal_alloc_cont_rdp(hal);
        if (ret)
                goto err_hal;

        ret = ath12k_hal_alloc_cont_wrp(hal);
        if (ret)
                goto err_free_cont_rdp;

        ath12k_hal_register_srng_lock_keys(hal);

        return 0;

err_free_cont_rdp:
        ath12k_hal_free_cont_rdp(hal);

err_hal:
        return ret;
}

void ath12k_hal_srng_deinit(struct ath12k_base *ab)
{
        struct ath12k_hal *hal = &ab->hal;

        ath12k_hal_unregister_srng_lock_keys(hal);
        ath12k_hal_free_cont_rdp(hal);
        ath12k_hal_free_cont_wrp(hal);
        kfree(hal->srng_config);
        hal->srng_config = NULL;
}

void ath12k_hal_dump_srng_stats(struct ath12k_base *ab)
{
        struct hal_srng *srng;
        struct ath12k_ext_irq_grp *irq_grp;
        struct ath12k_ce_pipe *ce_pipe;
        int i;

        ath12k_err(ab, "Last interrupt received for each CE:\n");
        for (i = 0; i < ab->hw_params->ce_count; i++) {
                ce_pipe = &ab->ce.ce_pipe[i];

                if (ath12k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
                        continue;

                ath12k_err(ab, "CE_id %d pipe_num %d %ums before\n",
                           i, ce_pipe->pipe_num,
                           jiffies_to_msecs(jiffies - ce_pipe->timestamp));
        }

        ath12k_err(ab, "\nLast interrupt received for each group:\n");
        for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) {
                irq_grp = &ab->ext_irq_grp[i];
                ath12k_err(ab, "group_id %d %ums before\n",
                           irq_grp->grp_id,
                           jiffies_to_msecs(jiffies - irq_grp->timestamp));
        }

        for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) {
                srng = &ab->hal.srng_list[i];

                if (!srng->initialized)
                        continue;

                if (srng->ring_dir == HAL_SRNG_DIR_SRC)
                        ath12k_err(ab,
                                   "src srng id %u hp %u, reap_hp %u, cur tp %u, cached tp %u last tp %u napi processed before %ums\n",
                                   srng->ring_id, srng->u.src_ring.hp,
                                   srng->u.src_ring.reap_hp,
                                   *srng->u.src_ring.tp_addr, srng->u.src_ring.cached_tp,
                                   srng->u.src_ring.last_tp,
                                   jiffies_to_msecs(jiffies - srng->timestamp));
                else if (srng->ring_dir == HAL_SRNG_DIR_DST)
                        ath12k_err(ab,
                                   "dst srng id %u tp %u, cur hp %u, cached hp %u last hp %u napi processed before %ums\n",
                                   srng->ring_id, srng->u.dst_ring.tp,
                                   *srng->u.dst_ring.hp_addr,
                                   srng->u.dst_ring.cached_hp,
                                   srng->u.dst_ring.last_hp,
                                   jiffies_to_msecs(jiffies - srng->timestamp));
        }
}

void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len)
{
        struct hal_tlv_64_hdr *tlv64 = tlv;

        tlv64->tl = le64_encode_bits(tag, HAL_TLV_HDR_TAG) |
                    le64_encode_bits(len, HAL_TLV_HDR_LEN);

        return tlv64->value;
}
EXPORT_SYMBOL(ath12k_hal_encode_tlv64_hdr);

void *ath12k_hal_encode_tlv32_hdr(void *tlv, u64 tag, u64 len)
{
        struct hal_tlv_hdr *tlv32 = tlv;

        tlv32->tl = le32_encode_bits(tag, HAL_TLV_HDR_TAG) |
                    le32_encode_bits(len, HAL_TLV_HDR_LEN);

        return tlv32->value;
}
EXPORT_SYMBOL(ath12k_hal_encode_tlv32_hdr);

u16 ath12k_hal_decode_tlv64_hdr(void *tlv, void **desc)
{
        struct hal_tlv_64_hdr *tlv64 = tlv;
        u16 tag;

        tag = le64_get_bits(tlv64->tl, HAL_SRNG_TLV_HDR_TAG);
        *desc = tlv64->value;

        return tag;
}
EXPORT_SYMBOL(ath12k_hal_decode_tlv64_hdr);

u16 ath12k_hal_decode_tlv32_hdr(void *tlv, void **desc)
{
        struct hal_tlv_hdr *tlv32 = tlv;
        u16 tag;

        tag = le32_get_bits(tlv32->tl, HAL_SRNG_TLV_HDR_TAG);
        *desc = tlv32->value;

        return tag;
}
EXPORT_SYMBOL(ath12k_hal_decode_tlv32_hdr);