// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2015-2018 Netronome Systems, Inc. */

/*
 * nfp_cppcore.c
 * Provides low-level access to the NFP's internal CPP bus
 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
 *          Jason McMullan <jason.mcmullan@netronome.com>
 *          Rolf Neugebauer <rolf.neugebauer@netronome.com>
 */

#include <linux/unaligned.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/wait.h>

#include "nfp_arm.h"
#include "nfp_cpp.h"
#include "nfp6000/nfp6000.h"

#define NFP_ARM_GCSR_SOFTMODEL2                              0x0000014c
#define NFP_ARM_GCSR_SOFTMODEL3                              0x00000150

struct nfp_cpp_resource {
        struct list_head list;
        const char *name;
        u32 cpp_id;
        u64 start;
        u64 end;
};

/**
 * struct nfp_cpp - main nfpcore device structure
 * Following fields are read-only after probe() exits or netdevs are spawned.
 * @dev:                embedded device structure
 * @op:                 low-level implementation ops
 * @priv:               private data of the low-level implementation
 * @model:              chip model
 * @interface:          chip interface id we are using to reach it
 * @serial:             chip serial number
 * @imb_cat_table:      CPP Mapping Table
 * @mu_locality_lsb:    MU access type bit offset
 *
 * Following fields use explicit locking:
 * @resource_list:      NFP CPP resource list
 * @resource_lock:      protects @resource_list
 *
 * @area_cache_list:    cached areas for cpp/xpb read/write speed up
 * @area_cache_mutex:   protects @area_cache_list
 *
 * @waitq:              area wait queue
 */
struct nfp_cpp {
        struct device dev;

        void *priv;

        u32 model;
        u16 interface;
        u8 serial[NFP_SERIAL_LEN];

        const struct nfp_cpp_operations *op;
        struct list_head resource_list;
        rwlock_t resource_lock;
        wait_queue_head_t waitq;

        u32 imb_cat_table[16];
        unsigned int mu_locality_lsb;

        struct mutex area_cache_mutex;
        struct list_head area_cache_list;
};

/* Element of the area_cache_list */
struct nfp_cpp_area_cache {
        struct list_head entry;
        u32 id;
        u64 addr;
        u32 size;
        struct nfp_cpp_area *area;
};

struct nfp_cpp_area {
        struct nfp_cpp *cpp;
        struct kref kref;
        atomic_t refcount;
        struct mutex mutex;     /* Lock for the area's refcount */
        unsigned long long offset;
        unsigned long size;
        struct nfp_cpp_resource resource;
        void __iomem *iomem;
        /* Here follows the 'priv' part of nfp_cpp_area. */
};

struct nfp_cpp_explicit {
        struct nfp_cpp *cpp;
        struct nfp_cpp_explicit_command cmd;
        /* Here follows the 'priv' part of nfp_cpp_area. */
};

static void __resource_add(struct list_head *head, struct nfp_cpp_resource *res)
{
        struct nfp_cpp_resource *tmp;
        struct list_head *pos;

        list_for_each(pos, head) {
                tmp = container_of(pos, struct nfp_cpp_resource, list);

                if (tmp->cpp_id > res->cpp_id)
                        break;

                if (tmp->cpp_id == res->cpp_id && tmp->start > res->start)
                        break;
        }

        list_add_tail(&res->list, pos);
}

static void __resource_del(struct nfp_cpp_resource *res)
{
        list_del_init(&res->list);
}

static void __release_cpp_area(struct kref *kref)
{
        struct nfp_cpp_area *area =
                container_of(kref, struct nfp_cpp_area, kref);
        struct nfp_cpp *cpp = nfp_cpp_area_cpp(area);

        if (area->cpp->op->area_cleanup)
                area->cpp->op->area_cleanup(area);

        write_lock(&cpp->resource_lock);
        __resource_del(&area->resource);
        write_unlock(&cpp->resource_lock);
        kfree(area);
}

static void nfp_cpp_area_put(struct nfp_cpp_area *area)
{
        kref_put(&area->kref, __release_cpp_area);
}

static struct nfp_cpp_area *nfp_cpp_area_get(struct nfp_cpp_area *area)
{
        kref_get(&area->kref);

        return area;
}

/**
 * nfp_cpp_free() - free the CPP handle
 * @cpp:        CPP handle
 */
void nfp_cpp_free(struct nfp_cpp *cpp)
{
        struct nfp_cpp_area_cache *cache, *ctmp;
        struct nfp_cpp_resource *res, *rtmp;

        /* Remove all caches */
        list_for_each_entry_safe(cache, ctmp, &cpp->area_cache_list, entry) {
                list_del(&cache->entry);
                if (cache->id)
                        nfp_cpp_area_release(cache->area);
                nfp_cpp_area_free(cache->area);
                kfree(cache);
        }

        /* There should be no dangling areas at this point */
        WARN_ON(!list_empty(&cpp->resource_list));

        /* .. but if they weren't, try to clean up. */
        list_for_each_entry_safe(res, rtmp, &cpp->resource_list, list) {
                struct nfp_cpp_area *area = container_of(res,
                                                         struct nfp_cpp_area,
                                                         resource);

                dev_err(cpp->dev.parent, "Dangling area: %d:%d:%d:0x%0llx-0x%0llx%s%s\n",
                        NFP_CPP_ID_TARGET_of(res->cpp_id),
                        NFP_CPP_ID_ACTION_of(res->cpp_id),
                        NFP_CPP_ID_TOKEN_of(res->cpp_id),
                        res->start, res->end,
                        res->name ? " " : "",
                        res->name ? res->name : "");

                if (area->cpp->op->area_release)
                        area->cpp->op->area_release(area);

                __release_cpp_area(&area->kref);
        }

        if (cpp->op->free)
                cpp->op->free(cpp);

        device_unregister(&cpp->dev);

        kfree(cpp);
}

/**
 * nfp_cpp_model() - Retrieve the Model ID of the NFP
 * @cpp:        NFP CPP handle
 *
 * Return: NFP CPP Model ID
 */
u32 nfp_cpp_model(struct nfp_cpp *cpp)
{
        return cpp->model;
}

/**
 * nfp_cpp_interface() - Retrieve the Interface ID of the NFP
 * @cpp:        NFP CPP handle
 *
 * Return: NFP CPP Interface ID
 */
u16 nfp_cpp_interface(struct nfp_cpp *cpp)
{
        return cpp->interface;
}

/**
 * nfp_cpp_serial() - Retrieve the Serial ID of the NFP
 * @cpp:        NFP CPP handle
 * @serial:     Pointer to NFP serial number
 *
 * Return:  Length of NFP serial number
 */
int nfp_cpp_serial(struct nfp_cpp *cpp, const u8 **serial)
{
        *serial = &cpp->serial[0];
        return sizeof(cpp->serial);
}

#define NFP_IMB_TGTADDRESSMODECFG_MODE_of(_x)           (((_x) >> 13) & 0x7)
#define NFP_IMB_TGTADDRESSMODECFG_ADDRMODE              BIT(12)
#define   NFP_IMB_TGTADDRESSMODECFG_ADDRMODE_32_BIT     0
#define   NFP_IMB_TGTADDRESSMODECFG_ADDRMODE_40_BIT     BIT(12)

static int nfp_cpp_set_mu_locality_lsb(struct nfp_cpp *cpp)
{
        unsigned int mode, addr40;
        u32 imbcppat;
        int res;

        imbcppat = cpp->imb_cat_table[NFP_CPP_TARGET_MU];
        mode = NFP_IMB_TGTADDRESSMODECFG_MODE_of(imbcppat);
        addr40 = !!(imbcppat & NFP_IMB_TGTADDRESSMODECFG_ADDRMODE);

        res = nfp_cppat_mu_locality_lsb(mode, addr40);
        if (res < 0)
                return res;
        cpp->mu_locality_lsb = res;

        return 0;
}

unsigned int nfp_cpp_mu_locality_lsb(struct nfp_cpp *cpp)
{
        return cpp->mu_locality_lsb;
}

/**
 * nfp_cpp_area_alloc_with_name() - allocate a new CPP area
 * @cpp:        CPP device handle
 * @dest:       NFP CPP ID
 * @name:       Name of region
 * @address:    Address of region
 * @size:       Size of region
 *
 * Allocate and initialize a CPP area structure.  The area must later
 * be locked down with an 'acquire' before it can be safely accessed.
 *
 * NOTE: @address and @size must be 32-bit aligned values.
 *
 * Return: NFP CPP area handle, or NULL
 */
struct nfp_cpp_area *
nfp_cpp_area_alloc_with_name(struct nfp_cpp *cpp, u32 dest, const char *name,
                             unsigned long long address, unsigned long size)
{
        struct nfp_cpp_area *area;
        u64 tmp64 = address;
        int err, name_len;

        /* Remap from cpp_island to cpp_target */
        err = nfp_target_cpp(dest, tmp64, &dest, &tmp64, cpp->imb_cat_table);
        if (err < 0)
                return NULL;

        address = tmp64;

        if (!name)
                name = "(reserved)";

        name_len = strlen(name) + 1;
        area = kzalloc(sizeof(*area) + cpp->op->area_priv_size + name_len,
                       GFP_KERNEL);
        if (!area)
                return NULL;

        area->cpp = cpp;
        area->resource.name = (void *)area + sizeof(*area) +
                cpp->op->area_priv_size;
        memcpy((char *)area->resource.name, name, name_len);

        area->resource.cpp_id = dest;
        area->resource.start = address;
        area->resource.end = area->resource.start + size - 1;
        INIT_LIST_HEAD(&area->resource.list);

        atomic_set(&area->refcount, 0);
        kref_init(&area->kref);
        mutex_init(&area->mutex);

        if (cpp->op->area_init) {
                int err;

                err = cpp->op->area_init(area, dest, address, size);
                if (err < 0) {
                        kfree(area);
                        return NULL;
                }
        }

        write_lock(&cpp->resource_lock);
        __resource_add(&cpp->resource_list, &area->resource);
        write_unlock(&cpp->resource_lock);

        area->offset = address;
        area->size = size;

        return area;
}

/**
 * nfp_cpp_area_alloc() - allocate a new CPP area
 * @cpp:        CPP handle
 * @dest:       CPP id
 * @address:    Start address on CPP target
 * @size:       Size of area in bytes
 *
 * Allocate and initialize a CPP area structure.  The area must later
 * be locked down with an 'acquire' before it can be safely accessed.
 *
 * NOTE: @address and @size must be 32-bit aligned values.
 *
 * Return: NFP CPP Area handle, or NULL
 */
struct nfp_cpp_area *
nfp_cpp_area_alloc(struct nfp_cpp *cpp, u32 dest,
                   unsigned long long address, unsigned long size)
{
        return nfp_cpp_area_alloc_with_name(cpp, dest, NULL, address, size);
}

/**
 * nfp_cpp_area_alloc_acquire() - allocate a new CPP area and lock it down
 * @cpp:        CPP handle
 * @name:       Name of region
 * @dest:       CPP id
 * @address:    Start address on CPP target
 * @size:       Size of area
 *
 * Allocate and initialize a CPP area structure, and lock it down so
 * that it can be accessed directly.
 *
 * NOTE: @address and @size must be 32-bit aligned values.
 * The area must also be 'released' when the structure is freed.
 *
 * Return: NFP CPP Area handle, or NULL
 */
struct nfp_cpp_area *
nfp_cpp_area_alloc_acquire(struct nfp_cpp *cpp, const char *name, u32 dest,
                           unsigned long long address, unsigned long size)
{
        struct nfp_cpp_area *area;

        area = nfp_cpp_area_alloc_with_name(cpp, dest, name, address, size);
        if (!area)
                return NULL;

        if (nfp_cpp_area_acquire(area)) {
                nfp_cpp_area_free(area);
                return NULL;
        }

        return area;
}

/**
 * nfp_cpp_area_free() - free up the CPP area
 * @area:       CPP area handle
 *
 * Frees up memory resources held by the CPP area.
 */
void nfp_cpp_area_free(struct nfp_cpp_area *area)
{
        if (atomic_read(&area->refcount))
                nfp_warn(area->cpp, "Warning: freeing busy area\n");
        nfp_cpp_area_put(area);
}

static bool nfp_cpp_area_acquire_try(struct nfp_cpp_area *area, int *status)
{
        *status = area->cpp->op->area_acquire(area);

        return *status != -EAGAIN;
}

static int __nfp_cpp_area_acquire(struct nfp_cpp_area *area)
{
        int err, status;

        if (atomic_inc_return(&area->refcount) > 1)
                return 0;

        if (!area->cpp->op->area_acquire)
                return 0;

        err = wait_event_interruptible(area->cpp->waitq,
                                       nfp_cpp_area_acquire_try(area, &status));
        if (!err)
                err = status;
        if (err) {
                nfp_warn(area->cpp, "Warning: area wait failed: %d\n", err);
                atomic_dec(&area->refcount);
                return err;
        }

        nfp_cpp_area_get(area);

        return 0;
}

/**
 * nfp_cpp_area_acquire() - lock down a CPP area for access
 * @area:       CPP area handle
 *
 * Locks down the CPP area for a potential long term activity.  Area
 * must always be locked down before being accessed.
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_area_acquire(struct nfp_cpp_area *area)
{
        int ret;

        mutex_lock(&area->mutex);
        ret = __nfp_cpp_area_acquire(area);
        mutex_unlock(&area->mutex);

        return ret;
}

/**
 * nfp_cpp_area_acquire_nonblocking() - lock down a CPP area for access
 * @area:       CPP area handle
 *
 * Locks down the CPP area for a potential long term activity.  Area
 * must always be locked down before being accessed.
 *
 * NOTE: Returns -EAGAIN is no area is available
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_area_acquire_nonblocking(struct nfp_cpp_area *area)
{
        mutex_lock(&area->mutex);
        if (atomic_inc_return(&area->refcount) == 1) {
                if (area->cpp->op->area_acquire) {
                        int err;

                        err = area->cpp->op->area_acquire(area);
                        if (err < 0) {
                                atomic_dec(&area->refcount);
                                mutex_unlock(&area->mutex);
                                return err;
                        }
                }
        }
        mutex_unlock(&area->mutex);

        nfp_cpp_area_get(area);
        return 0;
}

/**
 * nfp_cpp_area_release() - release a locked down CPP area
 * @area:       CPP area handle
 *
 * Releases a previously locked down CPP area.
 */
void nfp_cpp_area_release(struct nfp_cpp_area *area)
{
        mutex_lock(&area->mutex);
        /* Only call the release on refcount == 0 */
        if (atomic_dec_and_test(&area->refcount)) {
                if (area->cpp->op->area_release) {
                        area->cpp->op->area_release(area);
                        /* Let anyone waiting for a BAR try to get one.. */
                        wake_up_interruptible_all(&area->cpp->waitq);
                }
        }
        mutex_unlock(&area->mutex);

        nfp_cpp_area_put(area);
}

/**
 * nfp_cpp_area_release_free() - release CPP area and free it
 * @area:       CPP area handle
 *
 * Releases CPP area and frees up memory resources held by the it.
 */
void nfp_cpp_area_release_free(struct nfp_cpp_area *area)
{
        nfp_cpp_area_release(area);
        nfp_cpp_area_free(area);
}

/**
 * nfp_cpp_area_read() - read data from CPP area
 * @area:         CPP area handle
 * @offset:       offset into CPP area
 * @kernel_vaddr: kernel address to put data into
 * @length:       number of bytes to read
 *
 * Read data from indicated CPP region.
 *
 * NOTE: @offset and @length must be 32-bit aligned values.
 * Area must have been locked down with an 'acquire'.
 *
 * Return: length of io, or -ERRNO
 */
int nfp_cpp_area_read(struct nfp_cpp_area *area,
                      unsigned long offset, void *kernel_vaddr,
                      size_t length)
{
        return area->cpp->op->area_read(area, kernel_vaddr, offset, length);
}

/**
 * nfp_cpp_area_write() - write data to CPP area
 * @area:       CPP area handle
 * @offset:     offset into CPP area
 * @kernel_vaddr: kernel address to read data from
 * @length:     number of bytes to write
 *
 * Write data to indicated CPP region.
 *
 * NOTE: @offset and @length must be 32-bit aligned values.
 * Area must have been locked down with an 'acquire'.
 *
 * Return: length of io, or -ERRNO
 */
int nfp_cpp_area_write(struct nfp_cpp_area *area,
                       unsigned long offset, const void *kernel_vaddr,
                       size_t length)
{
        return area->cpp->op->area_write(area, kernel_vaddr, offset, length);
}

/**
 * nfp_cpp_area_size() - return size of a CPP area
 * @cpp_area:   CPP area handle
 *
 * Return: Size of the area
 */
size_t nfp_cpp_area_size(struct nfp_cpp_area *cpp_area)
{
        return cpp_area->size;
}

/**
 * nfp_cpp_area_name() - return name of a CPP area
 * @cpp_area:   CPP area handle
 *
 * Return: Name of the area, or NULL
 */
const char *nfp_cpp_area_name(struct nfp_cpp_area *cpp_area)
{
        return cpp_area->resource.name;
}

/**
 * nfp_cpp_area_priv() - return private struct for CPP area
 * @cpp_area:   CPP area handle
 *
 * Return: Private data for the CPP area
 */
void *nfp_cpp_area_priv(struct nfp_cpp_area *cpp_area)
{
        return &cpp_area[1];
}

/**
 * nfp_cpp_area_cpp() - return CPP handle for CPP area
 * @cpp_area:   CPP area handle
 *
 * Return: NFP CPP handle
 */
struct nfp_cpp *nfp_cpp_area_cpp(struct nfp_cpp_area *cpp_area)
{
        return cpp_area->cpp;
}

/**
 * nfp_cpp_area_resource() - get resource
 * @area:       CPP area handle
 *
 * NOTE: Area must have been locked down with an 'acquire'.
 *
 * Return: struct resource pointer, or NULL
 */
struct resource *nfp_cpp_area_resource(struct nfp_cpp_area *area)
{
        struct resource *res = NULL;

        if (area->cpp->op->area_resource)
                res = area->cpp->op->area_resource(area);

        return res;
}

/**
 * nfp_cpp_area_phys() - get physical address of CPP area
 * @area:       CPP area handle
 *
 * NOTE: Area must have been locked down with an 'acquire'.
 *
 * Return: phy_addr_t of the area, or NULL
 */
phys_addr_t nfp_cpp_area_phys(struct nfp_cpp_area *area)
{
        phys_addr_t addr = ~0;

        if (area->cpp->op->area_phys)
                addr = area->cpp->op->area_phys(area);

        return addr;
}

/**
 * nfp_cpp_area_iomem() - get IOMEM region for CPP area
 * @area:       CPP area handle
 *
 * Returns an iomem pointer for use with readl()/writel() style
 * operations.
 *
 * NOTE: Area must have been locked down with an 'acquire'.
 *
 * Return: __iomem pointer to the area, or NULL
 */
void __iomem *nfp_cpp_area_iomem(struct nfp_cpp_area *area)
{
        void __iomem *iomem = NULL;

        if (area->cpp->op->area_iomem)
                iomem = area->cpp->op->area_iomem(area);

        return iomem;
}

/**
 * nfp_cpp_area_readl() - Read a u32 word from an area
 * @area:       CPP Area handle
 * @offset:     Offset into area
 * @value:      Pointer to read buffer
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_cpp_area_readl(struct nfp_cpp_area *area,
                       unsigned long offset, u32 *value)
{
        u8 tmp[4];
        int n;

        n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
        if (n != sizeof(tmp))
                return n < 0 ? n : -EIO;

        *value = get_unaligned_le32(tmp);
        return 0;
}

/**
 * nfp_cpp_area_writel() - Write a u32 word to an area
 * @area:       CPP Area handle
 * @offset:     Offset into area
 * @value:      Value to write
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_cpp_area_writel(struct nfp_cpp_area *area,
                        unsigned long offset, u32 value)
{
        u8 tmp[4];
        int n;

        put_unaligned_le32(value, tmp);
        n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));

        return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
}

/**
 * nfp_cpp_area_readq() - Read a u64 word from an area
 * @area:       CPP Area handle
 * @offset:     Offset into area
 * @value:      Pointer to read buffer
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_cpp_area_readq(struct nfp_cpp_area *area,
                       unsigned long offset, u64 *value)
{
        u8 tmp[8];
        int n;

        n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
        if (n != sizeof(tmp))
                return n < 0 ? n : -EIO;

        *value = get_unaligned_le64(tmp);
        return 0;
}

/**
 * nfp_cpp_area_writeq() - Write a u64 word to an area
 * @area:       CPP Area handle
 * @offset:     Offset into area
 * @value:      Value to write
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_cpp_area_writeq(struct nfp_cpp_area *area,
                        unsigned long offset, u64 value)
{
        u8 tmp[8];
        int n;

        put_unaligned_le64(value, tmp);
        n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));

        return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
}

/**
 * nfp_cpp_area_fill() - fill a CPP area with a value
 * @area:       CPP area
 * @offset:     offset into CPP area
 * @value:      value to fill with
 * @length:     length of area to fill
 *
 * Fill indicated area with given value.
 *
 * Return: length of io, or -ERRNO
 */
int nfp_cpp_area_fill(struct nfp_cpp_area *area,
                      unsigned long offset, u32 value, size_t length)
{
        u8 tmp[4];
        size_t i;
        int k;

        put_unaligned_le32(value, tmp);

        if (offset % sizeof(tmp) || length % sizeof(tmp))
                return -EINVAL;

        for (i = 0; i < length; i += sizeof(tmp)) {
                k = nfp_cpp_area_write(area, offset + i, &tmp, sizeof(tmp));
                if (k < 0)
                        return k;
        }

        return i;
}

/**
 * nfp_cpp_area_cache_add() - Permanently reserve and area for the hot cache
 * @cpp:        NFP CPP handle
 * @size:       Size of the area - MUST BE A POWER OF 2.
 */
int nfp_cpp_area_cache_add(struct nfp_cpp *cpp, size_t size)
{
        struct nfp_cpp_area_cache *cache;
        struct nfp_cpp_area *area;

        /* Allocate an area - we use the MU target's base as a placeholder,
         * as all supported chips have a MU.
         */
        area = nfp_cpp_area_alloc(cpp, NFP_CPP_ID(7, NFP_CPP_ACTION_RW, 0),
                                  0, size);
        if (!area)
                return -ENOMEM;

        cache = kzalloc_obj(*cache);
        if (!cache) {
                nfp_cpp_area_free(area);
                return -ENOMEM;
        }

        cache->id = 0;
        cache->addr = 0;
        cache->size = size;
        cache->area = area;
        mutex_lock(&cpp->area_cache_mutex);
        list_add_tail(&cache->entry, &cpp->area_cache_list);
        mutex_unlock(&cpp->area_cache_mutex);

        return 0;
}

static struct nfp_cpp_area_cache *
area_cache_get(struct nfp_cpp *cpp, u32 id,
               u64 addr, unsigned long *offset, size_t length)
{
        struct nfp_cpp_area_cache *cache;
        int err;

        /* Early exit when length == 0, which prevents
         * the need for special case code below when
         * checking against available cache size.
         */
        if (length == 0 || id == 0)
                return NULL;

        /* Remap from cpp_island to cpp_target */
        err = nfp_target_cpp(id, addr, &id, &addr, cpp->imb_cat_table);
        if (err < 0)
                return NULL;

        mutex_lock(&cpp->area_cache_mutex);

        if (list_empty(&cpp->area_cache_list)) {
                mutex_unlock(&cpp->area_cache_mutex);
                return NULL;
        }

        addr += *offset;

        /* See if we have a match */
        list_for_each_entry(cache, &cpp->area_cache_list, entry) {
                if (id == cache->id &&
                    addr >= cache->addr &&
                    addr + length <= cache->addr + cache->size)
                        goto exit;
        }

        /* No matches - inspect the tail of the LRU */
        cache = list_entry(cpp->area_cache_list.prev,
                           struct nfp_cpp_area_cache, entry);

        /* Can we fit in the cache entry? */
        if (round_down(addr + length - 1, cache->size) !=
            round_down(addr, cache->size)) {
                mutex_unlock(&cpp->area_cache_mutex);
                return NULL;
        }

        /* If id != 0, we will need to release it */
        if (cache->id) {
                nfp_cpp_area_release(cache->area);
                cache->id = 0;
                cache->addr = 0;
        }

        /* Adjust the start address to be cache size aligned */
        cache->addr = addr & ~(u64)(cache->size - 1);

        /* Re-init to the new ID and address */
        if (cpp->op->area_init) {
                err = cpp->op->area_init(cache->area,
                                         id, cache->addr, cache->size);
                if (err < 0) {
                        mutex_unlock(&cpp->area_cache_mutex);
                        return NULL;
                }
        }

        /* Attempt to acquire */
        err = nfp_cpp_area_acquire(cache->area);
        if (err < 0) {
                mutex_unlock(&cpp->area_cache_mutex);
                return NULL;
        }

        cache->id = id;

exit:
        /* Adjust offset */
        *offset = addr - cache->addr;
        return cache;
}

static void
area_cache_put(struct nfp_cpp *cpp, struct nfp_cpp_area_cache *cache)
{
        if (!cache)
                return;

        /* Move to front of LRU */
        list_move(&cache->entry, &cpp->area_cache_list);

        mutex_unlock(&cpp->area_cache_mutex);
}

static int __nfp_cpp_read(struct nfp_cpp *cpp, u32 destination,
                          unsigned long long address, void *kernel_vaddr,
                          size_t length)
{
        struct nfp_cpp_area_cache *cache;
        struct nfp_cpp_area *area;
        unsigned long offset = 0;
        int err;

        cache = area_cache_get(cpp, destination, address, &offset, length);
        if (cache) {
                area = cache->area;
        } else {
                area = nfp_cpp_area_alloc(cpp, destination, address, length);
                if (!area)
                        return -ENOMEM;

                err = nfp_cpp_area_acquire(area);
                if (err) {
                        nfp_cpp_area_free(area);
                        return err;
                }
        }

        err = nfp_cpp_area_read(area, offset, kernel_vaddr, length);

        if (cache)
                area_cache_put(cpp, cache);
        else
                nfp_cpp_area_release_free(area);

        return err;
}

/**
 * nfp_cpp_read() - read from CPP target
 * @cpp:                CPP handle
 * @destination:        CPP id
 * @address:            offset into CPP target
 * @kernel_vaddr:       kernel buffer for result
 * @length:             number of bytes to read
 *
 * Return: length of io, or -ERRNO
 */
int nfp_cpp_read(struct nfp_cpp *cpp, u32 destination,
                 unsigned long long address, void *kernel_vaddr,
                 size_t length)
{
        size_t n, offset;
        int ret;

        for (offset = 0; offset < length; offset += n) {
                unsigned long long r_addr = address + offset;

                /* make first read smaller to align to safe window */
                n = min_t(size_t, length - offset,
                          ALIGN(r_addr + 1, NFP_CPP_SAFE_AREA_SIZE) - r_addr);

                ret = __nfp_cpp_read(cpp, destination, address + offset,
                                     kernel_vaddr + offset, n);
                if (ret < 0)
                        return ret;
                if (ret != n)
                        return offset + n;
        }

        return length;
}

static int __nfp_cpp_write(struct nfp_cpp *cpp, u32 destination,
                           unsigned long long address,
                           const void *kernel_vaddr, size_t length)
{
        struct nfp_cpp_area_cache *cache;
        struct nfp_cpp_area *area;
        unsigned long offset = 0;
        int err;

        cache = area_cache_get(cpp, destination, address, &offset, length);
        if (cache) {
                area = cache->area;
        } else {
                area = nfp_cpp_area_alloc(cpp, destination, address, length);
                if (!area)
                        return -ENOMEM;

                err = nfp_cpp_area_acquire(area);
                if (err) {
                        nfp_cpp_area_free(area);
                        return err;
                }
        }

        err = nfp_cpp_area_write(area, offset, kernel_vaddr, length);

        if (cache)
                area_cache_put(cpp, cache);
        else
                nfp_cpp_area_release_free(area);

        return err;
}

/**
 * nfp_cpp_write() - write to CPP target
 * @cpp:                CPP handle
 * @destination:        CPP id
 * @address:            offset into CPP target
 * @kernel_vaddr:       kernel buffer to read from
 * @length:             number of bytes to write
 *
 * Return: length of io, or -ERRNO
 */
int nfp_cpp_write(struct nfp_cpp *cpp, u32 destination,
                  unsigned long long address,
                  const void *kernel_vaddr, size_t length)
{
        size_t n, offset;
        int ret;

        for (offset = 0; offset < length; offset += n) {
                unsigned long long w_addr = address + offset;

                /* make first write smaller to align to safe window */
                n = min_t(size_t, length - offset,
                          ALIGN(w_addr + 1, NFP_CPP_SAFE_AREA_SIZE) - w_addr);

                ret = __nfp_cpp_write(cpp, destination, address + offset,
                                      kernel_vaddr + offset, n);
                if (ret < 0)
                        return ret;
                if (ret != n)
                        return offset + n;
        }

        return length;
}

/* Return the correct CPP address, and fixup xpb_addr as needed. */
static u32 nfp_xpb_to_cpp(struct nfp_cpp *cpp, u32 *xpb_addr)
{
        int island;
        u32 xpb;

        xpb = NFP_CPP_ID(14, NFP_CPP_ACTION_RW, 0);
        /* Ensure that non-local XPB accesses go
         * out through the global XPBM bus.
         */
        island = (*xpb_addr >> 24) & 0x3f;
        if (!island)
                return xpb;

        if (island != 1) {
                *xpb_addr |= 1 << 30;
                return xpb;
        }

        /* Accesses to the ARM Island overlay uses Island 0 / Global Bit */
        *xpb_addr &= ~0x7f000000;
        if (*xpb_addr < 0x60000) {
                *xpb_addr |= 1 << 30;
        } else {
                /* And only non-ARM interfaces use the island id = 1 */
                if (NFP_CPP_INTERFACE_TYPE_of(nfp_cpp_interface(cpp))
                    != NFP_CPP_INTERFACE_TYPE_ARM)
                        *xpb_addr |= 1 << 24;
        }

        return xpb;
}

/**
 * nfp_xpb_readl() - Read a u32 word from a XPB location
 * @cpp:        CPP device handle
 * @xpb_addr:   Address for operation
 * @value:      Pointer to read buffer
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_addr, u32 *value)
{
        u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr);

        return nfp_cpp_readl(cpp, cpp_dest, xpb_addr, value);
}

/**
 * nfp_xpb_writel() - Write a u32 word to a XPB location
 * @cpp:        CPP device handle
 * @xpb_addr:   Address for operation
 * @value:      Value to write
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_addr, u32 value)
{
        u32 cpp_dest = nfp_xpb_to_cpp(cpp, &xpb_addr);

        return nfp_cpp_writel(cpp, cpp_dest, xpb_addr, value);
}

/**
 * nfp_xpb_writelm() - Modify bits of a 32-bit value from the XPB bus
 * @cpp:        NFP CPP device handle
 * @xpb_tgt:    XPB target and address
 * @mask:       mask of bits to alter
 * @value:      value to modify
 *
 * KERNEL: This operation is safe to call in interrupt or softirq context.
 *
 * Return: 0 on success, or -ERRNO
 */
int nfp_xpb_writelm(struct nfp_cpp *cpp, u32 xpb_tgt,
                    u32 mask, u32 value)
{
        int err;
        u32 tmp;

        err = nfp_xpb_readl(cpp, xpb_tgt, &tmp);
        if (err < 0)
                return err;

        tmp &= ~mask;
        tmp |= mask & value;
        return nfp_xpb_writel(cpp, xpb_tgt, tmp);
}

/* Lockdep markers */
static struct lock_class_key nfp_cpp_resource_lock_key;

static void nfp_cpp_dev_release(struct device *dev)
{
        /* Nothing to do here - it just makes the kernel happy */
}

/**
 * nfp_cpp_from_operations() - Create a NFP CPP handle
 *                             from an operations structure
 * @ops:        NFP CPP operations structure
 * @parent:     Parent device
 * @priv:       Private data of low-level implementation
 *
 * NOTE: On failure, cpp_ops->free will be called!
 *
 * Return: NFP CPP handle on success, ERR_PTR on failure
 */
struct nfp_cpp *
nfp_cpp_from_operations(const struct nfp_cpp_operations *ops,
                        struct device *parent, void *priv)
{
        const u32 arm = NFP_CPP_ID(NFP_CPP_TARGET_ARM, NFP_CPP_ACTION_RW, 0);
        struct nfp_cpp *cpp;
        int ifc, err;
        u32 mask[2];
        u32 xpbaddr;
        size_t tgt;

        cpp = kzalloc_obj(*cpp);
        if (!cpp) {
                err = -ENOMEM;
                goto err_malloc;
        }

        cpp->op = ops;
        cpp->priv = priv;

        ifc = ops->get_interface(parent);
        if (ifc < 0) {
                err = ifc;
                goto err_free_cpp;
        }
        cpp->interface = ifc;
        if (ops->read_serial) {
                err = ops->read_serial(parent, cpp->serial);
                if (err)
                        goto err_free_cpp;
        }

        rwlock_init(&cpp->resource_lock);
        init_waitqueue_head(&cpp->waitq);
        lockdep_set_class(&cpp->resource_lock, &nfp_cpp_resource_lock_key);
        INIT_LIST_HEAD(&cpp->resource_list);
        INIT_LIST_HEAD(&cpp->area_cache_list);
        mutex_init(&cpp->area_cache_mutex);
        cpp->dev.init_name = "cpp";
        cpp->dev.parent = parent;
        cpp->dev.release = nfp_cpp_dev_release;
        err = device_register(&cpp->dev);
        if (err < 0) {
                put_device(&cpp->dev);
                goto err_free_cpp;
        }

        dev_set_drvdata(&cpp->dev, cpp);

        /* NOTE: cpp_lock is NOT locked for op->init,
         * since it may call NFP CPP API operations
         */
        if (cpp->op->init) {
                err = cpp->op->init(cpp);
                if (err < 0) {
                        dev_err(parent,
                                "NFP interface initialization failed\n");
                        goto err_out;
                }
        }

        err = nfp_cpp_model_autodetect(cpp, &cpp->model);
        if (err < 0) {
                dev_err(parent, "NFP model detection failed\n");
                goto err_out;
        }

        for (tgt = 0; tgt < ARRAY_SIZE(cpp->imb_cat_table); tgt++) {
                        /* Hardcoded XPB IMB Base, island 0 */
                xpbaddr = 0x000a0000 + (tgt * 4);
                err = nfp_xpb_readl(cpp, xpbaddr,
                                    &cpp->imb_cat_table[tgt]);
                if (err < 0) {
                        dev_err(parent,
                                "Can't read CPP mapping from device\n");
                        goto err_out;
                }
        }

        nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL2,
                      &mask[0]);
        nfp_cpp_readl(cpp, arm, NFP_ARM_GCSR + NFP_ARM_GCSR_SOFTMODEL3,
                      &mask[1]);

        err = nfp_cpp_set_mu_locality_lsb(cpp);
        if (err < 0) {
                dev_err(parent, "Can't calculate MU locality bit offset\n");
                goto err_out;
        }

        dev_info(cpp->dev.parent, "Model: 0x%08x, SN: %pM, Ifc: 0x%04x\n",
                 nfp_cpp_model(cpp), cpp->serial, nfp_cpp_interface(cpp));

        return cpp;

err_out:
        device_unregister(&cpp->dev);
err_free_cpp:
        kfree(cpp);
err_malloc:
        return ERR_PTR(err);
}

/**
 * nfp_cpp_priv() - Get the operations private data of a CPP handle
 * @cpp:        CPP handle
 *
 * Return: Private data for the NFP CPP handle
 */
void *nfp_cpp_priv(struct nfp_cpp *cpp)
{
        return cpp->priv;
}

/**
 * nfp_cpp_device() - Get the Linux device handle of a CPP handle
 * @cpp:        CPP handle
 *
 * Return: Device for the NFP CPP bus
 */
struct device *nfp_cpp_device(struct nfp_cpp *cpp)
{
        return &cpp->dev;
}

#define NFP_EXPL_OP(func, expl, args...)                          \
        ({                                                        \
                struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \
                int err = -ENODEV;                                \
                                                                  \
                if (cpp->op->func)                                \
                        err = cpp->op->func(expl, ##args);        \
                err;                                              \
        })

#define NFP_EXPL_OP_NR(func, expl, args...)                       \
        ({                                                        \
                struct nfp_cpp *cpp = nfp_cpp_explicit_cpp(expl); \
                                                                  \
                if (cpp->op->func)                                \
                        cpp->op->func(expl, ##args);              \
                                                                  \
        })

/**
 * nfp_cpp_explicit_acquire() - Acquire explicit access handle
 * @cpp:        NFP CPP handle
 *
 * The 'data_ref' and 'signal_ref' values are useful when
 * constructing the NFP_EXPL_CSR1 and NFP_EXPL_POST values.
 *
 * Return: NFP CPP explicit handle
 */
struct nfp_cpp_explicit *nfp_cpp_explicit_acquire(struct nfp_cpp *cpp)
{
        struct nfp_cpp_explicit *expl;
        int err;

        expl = kzalloc(sizeof(*expl) + cpp->op->explicit_priv_size, GFP_KERNEL);
        if (!expl)
                return NULL;

        expl->cpp = cpp;
        err = NFP_EXPL_OP(explicit_acquire, expl);
        if (err < 0) {
                kfree(expl);
                return NULL;
        }

        return expl;
}

/**
 * nfp_cpp_explicit_set_target() - Set target fields for explicit
 * @expl:       Explicit handle
 * @cpp_id:     CPP ID field
 * @len:        CPP Length field
 * @mask:       CPP Mask field
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_set_target(struct nfp_cpp_explicit *expl,
                                u32 cpp_id, u8 len, u8 mask)
{
        expl->cmd.cpp_id = cpp_id;
        expl->cmd.len = len;
        expl->cmd.byte_mask = mask;

        return 0;
}

/**
 * nfp_cpp_explicit_set_data() - Set data fields for explicit
 * @expl:       Explicit handle
 * @data_master: CPP Data Master field
 * @data_ref:   CPP Data Ref field
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_set_data(struct nfp_cpp_explicit *expl,
                              u8 data_master, u16 data_ref)
{
        expl->cmd.data_master = data_master;
        expl->cmd.data_ref = data_ref;

        return 0;
}

/**
 * nfp_cpp_explicit_set_signal() - Set signal fields for explicit
 * @expl:       Explicit handle
 * @signal_master: CPP Signal Master field
 * @signal_ref: CPP Signal Ref field
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_set_signal(struct nfp_cpp_explicit *expl,
                                u8 signal_master, u8 signal_ref)
{
        expl->cmd.signal_master = signal_master;
        expl->cmd.signal_ref = signal_ref;

        return 0;
}

/**
 * nfp_cpp_explicit_set_posted() - Set completion fields for explicit
 * @expl:       Explicit handle
 * @posted:     True for signaled completion, false otherwise
 * @siga:       CPP Signal A field
 * @siga_mode:  CPP Signal A Mode field
 * @sigb:       CPP Signal B field
 * @sigb_mode:  CPP Signal B Mode field
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_set_posted(struct nfp_cpp_explicit *expl, int posted,
                                u8 siga,
                                enum nfp_cpp_explicit_signal_mode siga_mode,
                                u8 sigb,
                                enum nfp_cpp_explicit_signal_mode sigb_mode)
{
        expl->cmd.posted = posted;
        expl->cmd.siga = siga;
        expl->cmd.sigb = sigb;
        expl->cmd.siga_mode = siga_mode;
        expl->cmd.sigb_mode = sigb_mode;

        return 0;
}

/**
 * nfp_cpp_explicit_put() - Set up the write (pull) data for a explicit access
 * @expl:       NFP CPP Explicit handle
 * @buff:       Data to have the target pull in the transaction
 * @len:        Length of data, in bytes
 *
 * The 'len' parameter must be less than or equal to 128 bytes.
 *
 * If this function is called before the configuration
 * registers are set, it will return -EINVAL.
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_put(struct nfp_cpp_explicit *expl,
                         const void *buff, size_t len)
{
        return NFP_EXPL_OP(explicit_put, expl, buff, len);
}

/**
 * nfp_cpp_explicit_do() - Execute a transaction, and wait for it to complete
 * @expl:       NFP CPP Explicit handle
 * @address:    Address to send in the explicit transaction
 *
 * If this function is called before the configuration
 * registers are set, it will return -1, with an errno of EINVAL.
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_do(struct nfp_cpp_explicit *expl, u64 address)
{
        return NFP_EXPL_OP(explicit_do, expl, &expl->cmd, address);
}

/**
 * nfp_cpp_explicit_get() - Get the 'push' (read) data from a explicit access
 * @expl:       NFP CPP Explicit handle
 * @buff:       Data that the target pushed in the transaction
 * @len:        Length of data, in bytes
 *
 * The 'len' parameter must be less than or equal to 128 bytes.
 *
 * If this function is called before all three configuration
 * registers are set, it will return -1, with an errno of EINVAL.
 *
 * If this function is called before nfp_cpp_explicit_do()
 * has completed, it will return -1, with an errno of EBUSY.
 *
 * Return: 0, or -ERRNO
 */
int nfp_cpp_explicit_get(struct nfp_cpp_explicit *expl, void *buff, size_t len)
{
        return NFP_EXPL_OP(explicit_get, expl, buff, len);
}

/**
 * nfp_cpp_explicit_release() - Release explicit access handle
 * @expl:       NFP CPP Explicit handle
 *
 */
void nfp_cpp_explicit_release(struct nfp_cpp_explicit *expl)
{
        NFP_EXPL_OP_NR(explicit_release, expl);
        kfree(expl);
}

/**
 * nfp_cpp_explicit_cpp() - return CPP handle for CPP explicit
 * @cpp_explicit:       CPP explicit handle
 *
 * Return: NFP CPP handle of the explicit
 */
struct nfp_cpp *nfp_cpp_explicit_cpp(struct nfp_cpp_explicit *cpp_explicit)
{
        return cpp_explicit->cpp;
}

/**
 * nfp_cpp_explicit_priv() - return private struct for CPP explicit
 * @cpp_explicit:       CPP explicit handle
 *
 * Return: private data of the explicit, or NULL
 */
void *nfp_cpp_explicit_priv(struct nfp_cpp_explicit *cpp_explicit)
{
        return &cpp_explicit[1];
}