/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SWIOTLB_H
#define __LINUX_SWIOTLB_H

#include <linux/device.h>
#include <linux/dma-direction.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/limits.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>

struct device;
struct page;
struct scatterlist;

#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */
#define SWIOTLB_FORCE   (1 << 1) /* force bounce buffering */
#define SWIOTLB_ANY     (1 << 2) /* allow any memory for the buffer */

/*
 * Maximum allowable number of contiguous slabs to map,
 * must be a power of 2.  What is the appropriate value ?
 * The complexity of {map,unmap}_single is linearly dependent on this value.
 */
#define IO_TLB_SEGSIZE  128

/*
 * log of the size of each IO TLB slab.  The number of slabs is command line
 * controllable.
 */
#define IO_TLB_SHIFT 11
#define IO_TLB_SIZE (1 << IO_TLB_SHIFT)

/* default to 64MB */
#define IO_TLB_DEFAULT_SIZE (64UL<<20)

unsigned long swiotlb_size_or_default(void);
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
        int (*remap)(void *tlb, unsigned long nslabs));
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
        int (*remap)(void *tlb, unsigned long nslabs));
extern void __init swiotlb_update_mem_attributes(void);

#ifdef CONFIG_SWIOTLB

/**
 * struct io_tlb_pool - IO TLB memory pool descriptor
 * @start:      The start address of the swiotlb memory pool. Used to do a quick
 *              range check to see if the memory was in fact allocated by this
 *              API.
 * @end:        The end address of the swiotlb memory pool. Used to do a quick
 *              range check to see if the memory was in fact allocated by this
 *              API.
 * @vaddr:      The vaddr of the swiotlb memory pool. The swiotlb memory pool
 *              may be remapped in the memory encrypted case and store virtual
 *              address for bounce buffer operation.
 * @nslabs:     The number of IO TLB slots between @start and @end. For the
 *              default swiotlb, this can be adjusted with a boot parameter,
 *              see setup_io_tlb_npages().
 * @late_alloc: %true if allocated using the page allocator.
 * @nareas:     Number of areas in the pool.
 * @area_nslabs: Number of slots in each area.
 * @areas:      Array of memory area descriptors.
 * @slots:      Array of slot descriptors.
 * @node:       Member of the IO TLB memory pool list.
 * @rcu:        RCU head for swiotlb_dyn_free().
 * @transient:  %true if transient memory pool.
 */
struct io_tlb_pool {
        phys_addr_t start;
        phys_addr_t end;
        void *vaddr;
        unsigned long nslabs;
        bool late_alloc;
        unsigned int nareas;
        unsigned int area_nslabs;
        struct io_tlb_area *areas;
        struct io_tlb_slot *slots;
#ifdef CONFIG_SWIOTLB_DYNAMIC
        struct list_head node;
        struct rcu_head rcu;
        bool transient;
#endif
};

/**
 * struct io_tlb_mem - Software IO TLB allocator
 * @defpool:    Default (initial) IO TLB memory pool descriptor.
 * @pool:       IO TLB memory pool descriptor (if not dynamic).
 * @nslabs:     Total number of IO TLB slabs in all pools.
 * @debugfs:    The dentry to debugfs.
 * @force_bounce: %true if swiotlb bouncing is forced
 * @for_alloc:  %true if the pool is used for memory allocation
 * @can_grow:   %true if more pools can be allocated dynamically.
 * @phys_limit: Maximum allowed physical address.
 * @lock:       Lock to synchronize changes to the list.
 * @pools:      List of IO TLB memory pool descriptors (if dynamic).
 * @dyn_alloc:  Dynamic IO TLB pool allocation work.
 * @total_used: The total number of slots in the pool that are currently used
 *              across all areas. Used only for calculating used_hiwater in
 *              debugfs.
 * @used_hiwater: The high water mark for total_used.  Used only for reporting
 *              in debugfs.
 * @transient_nslabs: The total number of slots in all transient pools that
 *              are currently used across all areas.
 */
struct io_tlb_mem {
        struct io_tlb_pool defpool;
        unsigned long nslabs;
        struct dentry *debugfs;
        bool force_bounce;
        bool for_alloc;
#ifdef CONFIG_SWIOTLB_DYNAMIC
        bool can_grow;
        u64 phys_limit;
        spinlock_t lock;
        struct list_head pools;
        struct work_struct dyn_alloc;
#endif
#ifdef CONFIG_DEBUG_FS
        atomic_long_t total_used;
        atomic_long_t used_hiwater;
        atomic_long_t transient_nslabs;
#endif
};

struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);

/**
 * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs
 * @dev:        Device which has mapped the buffer.
 * @paddr:      Physical address within the DMA buffer.
 *
 * Find the swiotlb pool that @paddr points into.
 *
 * Return:
 * * pool address if @paddr points into a bounce buffer
 * * NULL if @paddr does not point into a bounce buffer. As such, this function
 *   can be used to determine if @paddr denotes a swiotlb bounce buffer.
 */
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
                phys_addr_t paddr)
{
        struct io_tlb_mem *mem = dev->dma_io_tlb_mem;

        if (!mem)
                return NULL;

#ifdef CONFIG_SWIOTLB_DYNAMIC
        /*
         * All SWIOTLB buffer addresses must have been returned by
         * swiotlb_tbl_map_single() and passed to a device driver.
         * If a SWIOTLB address is checked on another CPU, then it was
         * presumably loaded by the device driver from an unspecified private
         * data structure. Make sure that this load is ordered before reading
         * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool().
         *
         * This barrier pairs with smp_mb() in swiotlb_find_slots().
         */
        smp_rmb();
        if (READ_ONCE(dev->dma_uses_io_tlb))
                return __swiotlb_find_pool(dev, paddr);
#else
        if (paddr >= mem->defpool.start && paddr < mem->defpool.end)
                return &mem->defpool;
#endif

        return NULL;
}

static inline bool is_swiotlb_force_bounce(struct device *dev)
{
        struct io_tlb_mem *mem = dev->dma_io_tlb_mem;

        return mem && mem->force_bounce;
}

void swiotlb_init(bool addressing_limited, unsigned int flags);
void __init swiotlb_exit(void);
void swiotlb_dev_init(struct device *dev);
size_t swiotlb_max_mapping_size(struct device *dev);
bool is_swiotlb_allocated(void);
bool is_swiotlb_active(struct device *dev);
void __init swiotlb_adjust_size(unsigned long size);
phys_addr_t default_swiotlb_base(void);
phys_addr_t default_swiotlb_limit(void);
#else
static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
{
}

static inline void swiotlb_dev_init(struct device *dev)
{
}

static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
                phys_addr_t paddr)
{
        return NULL;
}
static inline bool is_swiotlb_force_bounce(struct device *dev)
{
        return false;
}
static inline void swiotlb_exit(void)
{
}
static inline size_t swiotlb_max_mapping_size(struct device *dev)
{
        return SIZE_MAX;
}

static inline bool is_swiotlb_allocated(void)
{
        return false;
}

static inline bool is_swiotlb_active(struct device *dev)
{
        return false;
}

static inline void swiotlb_adjust_size(unsigned long size)
{
}

static inline phys_addr_t default_swiotlb_base(void)
{
        return 0;
}

static inline phys_addr_t default_swiotlb_limit(void)
{
        return 0;
}
#endif /* CONFIG_SWIOTLB */

phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
                size_t mapping_size, unsigned int alloc_aligned_mask,
                enum dma_data_direction dir, unsigned long attrs);
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
                size_t size, enum dma_data_direction dir, unsigned long attrs);

void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
                size_t mapping_size, enum dma_data_direction dir,
                unsigned long attrs, struct io_tlb_pool *pool);
static inline void swiotlb_tbl_unmap_single(struct device *dev,
                phys_addr_t addr, size_t size, enum dma_data_direction dir,
                unsigned long attrs)
{
        struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);

        if (unlikely(pool))
                __swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);
}

void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
                size_t size, enum dma_data_direction dir,
                struct io_tlb_pool *pool);
static inline void swiotlb_sync_single_for_device(struct device *dev,
                phys_addr_t addr, size_t size, enum dma_data_direction dir)
{
        struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);

        if (unlikely(pool))
                __swiotlb_sync_single_for_device(dev, addr, size, dir, pool);
}

void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
                size_t size, enum dma_data_direction dir,
                struct io_tlb_pool *pool);
static inline void swiotlb_sync_single_for_cpu(struct device *dev,
                phys_addr_t addr, size_t size, enum dma_data_direction dir)
{
        struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);

        if (unlikely(pool))
                __swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);
}

extern void swiotlb_print_info(void);

#ifdef CONFIG_DMA_RESTRICTED_POOL
struct page *swiotlb_alloc(struct device *dev, size_t size);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);

static inline bool is_swiotlb_for_alloc(struct device *dev)
{
        return dev->dma_io_tlb_mem->for_alloc;
}
#else
static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
{
        return NULL;
}
static inline bool swiotlb_free(struct device *dev, struct page *page,
                                size_t size)
{
        return false;
}
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
        return false;
}
#endif /* CONFIG_DMA_RESTRICTED_POOL */

#endif /* __LINUX_SWIOTLB_H */